def test_lvm_numbering(self): input_model = InputModel(config=safe_load(lvm_disk_model), hosts_fd=padawan_net_hosts, consumes=padawan_swf_rng_consumes) lv_devices = [] lv_info = set() for lv_device in input_model._iter_volume_groups(): lv_devices.append(DeviceInfo(lv_device)) lv_info.add((lv_device.server_name, lv_device.ring_name, lv_device.swift_drive_name)) self.assertEqual(len(lv_devices), 11) # Do not change this without also examining test_drivedata.py lv_expected = [('padawan-ccp-c1-m1', 'object-0', 'lvm0'), ('padawan-ccp-c1-m1', 'object-1', 'lvm0'), ('padawan-ccp-c1-m1', 'object-2', 'lvm1'), ('padawan-ccp-c1-m1', 'object-3', 'lvm1'), ('padawan-ccp-c1-m1', 'object-4', 'lvm2'), ('padawan-ccp-c1-m1', 'object-5', 'lvm2'), ('padawan-ccp-c1-m1', 'object-6', 'lvm2'), ('padawan-ccp-c1-m2', 'object-7', 'lvm0'), ('padawan-ccp-c1-m2', 'object-8', 'lvm1'), ('padawan-ccp-c1-m2', 'object-9', 'lvm3'), ('padawan-ccp-c1-m2', 'object-10', 'lvm3')] for lv in set(lv_expected): self.assertTrue(lv in lv_info, '%s missing from %s' % (lv, lv_info)) lv_expected.remove(lv) self.assertEqual(0, len(lv_expected), 'still have %s' % lv_expected)
def setUp(self): hosts = padawan_net_hosts = [ '192.168.245.2 padawan-ccp-c1-m3-mgmt', '192.168.222.2 padawan-ccp-c1-m3-obj', '192.168.245.3 padawan-ccp-c1-m2-mgmt', '192.168.222.3 padawan-ccp-c1-m2-obj', '192.168.245.4 padawan-ccp-c1-m1-mgmt', '192.168.222.4 padawan-ccp-c1-m1-obj', '192.168.111.4 padawan-vip'] self.input_model = InputModel(hosts_fd=hosts)
def test_add_servers(self): # This test uses same process as test_build_rings() above -- so it # appears as though all servers are being added to an existing # system. With --weight-step=10, the resulting weights are limited # to 10.0 options = DummyInputOptions() input_model = InputModel(config=safe_load(padawan_input_model), hosts_fd=padawan_net_hosts, consumes=padawan_swf_rng_consumes) ring_model = RingSpecifications(model=safe_load(padawan_input_model)) rings = RingBuilder(self.builder_dir, False) drive_configurations = dummy_osconfig_load( padawan_drive_configurations) # Set limit to weight options.weight_step = '10' delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) cmds = rebalance(delta, rings, True) self.assertTrue('--device disk0' ' --meta padawan-ccp-c1-m3:disk0:/dev/sdc' ' --weight 10.00' in ' '.join(cmds)) self.assertTrue('--device lvm0 --meta' ' padawan-ccp-c1-m2:lvm0:/dev/hlm-vg/LV_SWFAC' ' --weight 10.00' in ' '.join(cmds))
def test_set_weight_no_step(self): options = DummyInputOptions() input_model = InputModel(config=safe_load(padawan_input_model), hosts_fd=padawan_net_hosts, consumes=padawan_swf_rng_consumes) ring_model = RingSpecifications(model=safe_load(padawan_input_model)) rings = FakeRingBuilder(self.builder_dir, 'regionone', ['account', 'container', 'object-0'], 3.0) drive_configurations = dummy_osconfig_load( padawan_drive_configurations) delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) # Load the fake builder rings with the delta i.e., make it look as # though we had just done a rebalance() using input model rings.load_fake_ring_data(delta) # Change the weights to a small value rings.fake_set_weights(1.0) # make a new delta and rebalance delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) cmds = rebalance(delta, rings, True) self.assertTrue('account.builder set_weight' ' 192.168.245.2/disk0 18.63' in ' '.join(cmds))
def test_noop(self): options = DummyInputOptions() input_model = InputModel(config=safe_load(padawan_input_model), hosts_fd=padawan_net_hosts, consumes=padawan_swf_rng_consumes) ring_model = RingSpecifications(model=safe_load(padawan_input_model)) rings = FakeRingBuilder( self.builder_dir, 'regionone', ['account', 'container', 'object-0', 'object-1'], 3.0) drive_configurations = dummy_osconfig_load( padawan_drive_configurations) delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) # Load the fake builder rings with the delta i.e., make it look as # though we had just done a rebalance() using input model rings.load_fake_ring_data(delta) # make a new delta and rebalance delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) cmds = rebalance(delta, rings, True) # account, container, object-0: # 3 x rebalance (only) (which is point of this test) # object-1 has # 1 x set replica count # 2 x set weights # 1 x rebalance # total: 7 self.assertTrue(len(cmds) == 7)
def test_change_replica_count(self): options = DummyInputOptions() input_model = InputModel(config=safe_load(padawan_input_model), hosts_fd=padawan_net_hosts, consumes=padawan_swf_rng_consumes) ring_model = RingSpecifications(model=safe_load(padawan_input_model)) rings = FakeRingBuilder(self.builder_dir, 'regionone', ['container'], 4.0) drive_configurations = dummy_osconfig_load( padawan_drive_configurations) delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) cmds = rebalance(delta, rings, True) # Fake container ring has replica-count of 4.0, check that we # change it to match the model (3.0) self.assertTrue( verb_ringname_args_in_cmds('set_replicas', 'container.builder', ['3.0'], cmds)) # Validate we don't attempt to re-create container self.assertTrue(not verb_ringname_args_in_cmds( 'create', 'container.builder', None, cmds)) # Validate other rings are created self.assertTrue( verb_ringname_args_in_cmds('create', 'account.builder', ['17', '3.0', '24'], cmds)) self.assertTrue( verb_ringname_args_in_cmds('create', 'object-0.builder', ['17', '3.0', '24'], cmds))
def test_missmatched_region_names(self): options = DummyInputOptions() input_model = InputModel(config=safe_load(padawan_input_model), hosts_fd=padawan_net_hosts, consumes=padawan_swf_rng_consumes) ring_model = RingSpecifications(model=safe_load(padawan_input_model)) rings = RingBuilder(self.builder_dir, False) drive_configurations = dummy_osconfig_load( padawan_drive_configurations) delta = RingDelta() # Change region_name in rings for region_rings in ring_model.keystone_ring_specifications: region_rings.region_name = 'junk' self.assertRaises(SwiftModelException, generate_delta, input_model, ring_model, rings, drive_configurations, options, delta) # Put back original ring_model = RingSpecifications(model=safe_load(padawan_input_model)) # Add a region - but not used keystone_region_rings = KeystoneRegionRings(self) keystone_region_rings.region_name = 'unusedrregion' ring_model.keystone_ring_specifications.append(keystone_region_rings) # Does not stop options.stop_on_warnings = False generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) # Stops on warnings options.stop_on_warnings = True self.assertRaises(SwiftModelException, generate_delta, input_model, ring_model, rings, drive_configurations, options, delta)
def test_set_weight_down(self): options = DummyInputOptions() input_model = InputModel(config=safe_load(padawan_input_model), hosts_fd=padawan_net_hosts, consumes=padawan_swf_rng_consumes) ring_model = RingSpecifications(model=safe_load(padawan_input_model)) rings = FakeRingBuilder(self.builder_dir, 'regionone', ['account', 'container', 'object-0'], 3.0) drive_configurations = dummy_osconfig_load( padawan_drive_configurations) delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) # Load the fake builder rings with the delta i.e., make it look as # though we had just done a rebalance() using input model rings.load_fake_ring_data(delta) # Change the weights to a large value rings.fake_set_weights(30.0) # This make delta has a weight_step options.weight_step = '10.0' delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) cmds = rebalance(delta, rings, True) self.assertTrue('account.builder set_weight' ' 192.168.245.2/disk0 20.00' in ' '.join(cmds)) # Go through another cycle -- update as though last step built # the rings - use small step rings = FakeRingBuilder(self.builder_dir, 'regionone', ['account', 'container', 'object-0'], 3.0) rings.load_fake_ring_data(delta) options.weight_step = '1.0' delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) cmds = rebalance(delta, rings, True) self.assertTrue('account.builder set_weight' ' 192.168.245.2/disk0 19.00' in ' '.join(cmds)) # Go through another cycle -- the step is large enough that final # target weight is reached rings = FakeRingBuilder(self.builder_dir, 'regionone', ['account', 'container', 'object-0'], 3.0) rings.load_fake_ring_data(delta) options.weight_step = '10.0' delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) cmds = rebalance(delta, rings, True) self.assertTrue('account.builder set_weight' ' 192.168.245.2/disk0 18.63' in ' '.join(cmds))
def test_dev_numbering(self): input_model = InputModel(config=safe_load(device_groups_disk_model), hosts_fd=padawan_net_hosts, consumes=padawan_swf_rng_consumes) dev_devices = [] dev_info = set() for dev_device in input_model._iter_device_groups(): dev_devices.append(DeviceInfo(dev_device)) dev_info.add((dev_device.server_name, dev_device.ring_name, dev_device.swift_drive_name, dev_device.device_name)) self.assertEqual(len(dev_devices), 21) # Do not change this without also examining test_drivedata.py dev_expected = [ ('padawan-ccp-c1-m1', 'account', 'disk0', '/dev/sdb'), ('padawan-ccp-c1-m1', 'container', 'disk0', '/dev/sdb'), ('padawan-ccp-c1-m1', 'account', 'disk1', '/dev/sdc'), ('padawan-ccp-c1-m1', 'container', 'disk1', '/dev/sdc'), ('padawan-ccp-c1-m1', 'object-0', 'disk2', '/dev/sdd'), ('padawan-ccp-c1-m1', 'object-0', 'disk3', '/dev/sde'), ('padawan-ccp-c1-m1', 'object-0', 'disk4', '/dev/sdf'), ('padawan-ccp-c1-m1', 'object-1', 'disk5', '/dev/sdg'), ('padawan-ccp-c1-m1', 'object-1', 'disk6', '/dev/sdh'), ('padawan-ccp-c1-m1', 'object-1', 'disk7', '/dev/sdi'), ('padawan-ccp-c1-m1', 'object-1', 'disk11', '/dev/sdm'), ('padawan-ccp-c1-m1', 'object-0', 'disk11', '/dev/sdm'), ('padawan-ccp-c1-m1', 'object-1', 'disk12', '/dev/sdn'), ('padawan-ccp-c1-m1', 'object-0', 'disk12', '/dev/sdn'), ('padawan-ccp-c1-m1', 'object-1', 'disk13', '/dev/sdo'), ('padawan-ccp-c1-m1', 'object-0', 'disk13', '/dev/sdo'), ('padawan-ccp-c1-m2', 'account', 'disk0', '/dev/sda'), ('padawan-ccp-c1-m2', 'container', 'disk1', '/dev/sdb'), ('padawan-ccp-c1-m2', 'object-0', 'disk2', '/dev/sdc'), ('padawan-ccp-c1-m2', 'object-1', 'disk3', '/dev/sdd'), ('padawan-ccp-c1-m2', 'object-2', 'disk4', '/dev/sde') ] for dev in set(dev_expected): self.assertTrue(dev in dev_info, '%s missing from %s' % (dev, dev_info)) dev_info.remove(dev) self.assertEqual(0, len(dev_info), 'still have %s' % dev_info)
def test_iter_volume_groups(self): input_model = InputModel(config=safe_load(padawan_input_model), hosts_fd=padawan_net_hosts, consumes=padawan_swf_rng_consumes) lv_devices = [] lv_expected = [] for lv_device in input_model._iter_volume_groups(): lv_devices.append(DeviceInfo(lv_device)) self.assertEqual(len(lv_devices), 3) # This could be done with a simple assertEquals(), but this allowed # me to pin-point differences between actual and expected results for lv_device in expected_lv_devices: lv_expected.append(DeviceInfo(lv_device)) lv_expected = sorted(lv_expected, None, DeviceInfo.sortkey) lv_devices = sorted(lv_devices, None, DeviceInfo.sortkey) for i in range(0, len(lv_expected)): self.assertEqual(sorted(lv_expected[i].keys()), sorted(lv_devices[i].keys())) for key in lv_expected[i].keys(): self.assertEqual(lv_expected[i].get(key), lv_devices[i].get(key))
def test_build_rings(self): options = DummyInputOptions() input_model = InputModel(config=safe_load(padawan_input_model), hosts_fd=padawan_net_hosts, consumes=padawan_swf_rng_consumes) ring_model = RingSpecifications(model=safe_load(padawan_input_model)) rings = RingBuilder(self.builder_dir, False) drive_configurations = dummy_osconfig_load( padawan_drive_configurations) delta = RingDelta() generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) cmds = rebalance(delta, rings, True) assert_cmds_are_same(self, expected_cmds, cmds)
class TestIpAddress(unittest.TestCase): def setUp(self): hosts = padawan_net_hosts = [ '192.168.245.2 padawan-ccp-c1-m3-mgmt', '192.168.222.2 padawan-ccp-c1-m3-obj', '192.168.245.3 padawan-ccp-c1-m2-mgmt', '192.168.222.3 padawan-ccp-c1-m2-obj', '192.168.245.4 padawan-ccp-c1-m1-mgmt', '192.168.222.4 padawan-ccp-c1-m1-obj', '192.168.111.4 padawan-vip'] self.input_model = InputModel(hosts_fd=hosts) def TestMapping(self): self.assertEqual(self.input_model.ip_address('192.168.245.2', 'test'), 'padawan-ccp-c1-m3-mgmt') self.assertEqual(self.input_model.ip_address('padawan-ccp-c1-m3-mgmt', 'test'), '192.168.245.2') def TestNotFound(self): self.assertRaises(SwiftModelException, self.input_model.ip_address, 'junk', 'test') def TestAlias(self): expect = sorted(['192.168.245.2', '192.168.222.2']) alisas = self.input_model.aliases('192.168.245.2') self.assertEqual(sorted(alisas), expect) alisas = self.input_model.aliases('192.168.222.2') self.assertEqual(sorted(alisas), expect) expect = ['192.168.111.4'] alisas = self.input_model.aliases('192.168.111.4') self.assertEqual(sorted(alisas), expect) def TestBadAlias(self): alisas = self.input_model.aliases('junk') self.assertEqual(alisas, []) def TestEdgeCase(self): # As specified, the aliases only work for ip addresses -- returning # a list of ip addresses. However, given a valid name, it returns that # name -- not (by design) a list of names. That might be possible, # but not needed for now. alisas = self.input_model.aliases('padawan-ccp-c1-m3-mgmt') self.assertEqual(alisas, ['padawan-ccp-c1-m3-mgmt'])
def main(): parser = OptionParser(usage=usage) parser.add_option('--etc', dest='etc', default=DEFAULT_ETC, help='Overrides /etc/swiftlm (for testing)') parser.add_option('--input-vars', dest='input_vars', default=None, help='The name of yaml containing vars') parser.add_option('--builder-dir', dest='builder_dir', default=None, help='Builder file directory') parser.add_option('--osconfig', dest='osconfig', default=None, help='Directory where drive size data files exist.' ' If omitted, the correct weights cannot' ' be assigned to drives.') parser.add_option('--hosts', dest='hosts', default=None, help='Hosts file (/etc/hosts or ./net/hosts.hf)') parser.add_option('--swift-ring-builder-consumes', dest='swift_ring_builder_consumes', default=None, help='File containing the SWF_RNG variable') parser.add_option('--ring-delta', dest='ring_delta', default=None, help='Name of ring-delta file (as output or input' ' A value of "-" (on output means to write' ' to stdout') parser.add_option('--format', dest='fmt', default='yaml', help='One of yaml or json.' ' When used with --ring-delta, specifies the' ' format of the file.') parser.add_option('--detail', dest='detail', default='summary', help='Level of detail to use with --report.' ' Use summary or full') parser.add_option('--report', dest='report', default=False, action="store_true", help='Explain what the ring delta represents.' ' Optionally use --detail.') parser.add_option('--dry-run', dest='dry_run', default=False, action="store_true", help='Show the proposed swift-ring-builder commands') parser.add_option('--make-delta', dest='make_delta', default=False, action="store_true", help='Make a ring delta file') parser.add_option('--rebalance', dest='rebalance', default=False, action="store_true", help='Build (or rebalance) rings') parser.add_option('--storage-policies', dest='storage_policies', default=False, action="store_true", help='Extract storage policy data.' ' Use this to register the storage_policies' ' variable.') parser.add_option('--region-name', dest='region_name', default=None, help='Region name for use with --storage-policies') parser.add_option('--size-to-weight', dest='size_to_weight', default=float(1024 * 1024 * 1024), help='Conversion factor for size to weight. Default is' ' 1GB is weight of 1 (a 4Tb drive would be assigned' ' a weight of 4096') parser.add_option('--weight-step', dest='weight_step', default=None, help='When set, weights are changed by at most this' ' value. Overrides value in ring specification.') parser.add_option('--allow-partitions', dest='allow_partitions', default=False, action='store_true', help='Allow devices to be assigned to partitions.' ' Default is to use a full disk drive.') parser.add_option('--stop-on-warnings', dest='stop_on_warnings', default=False, action='store_true', help='Used with --make-delta. Exit with error if there' ' are model missmatch warnings.' ' Default is to only exit with error for errors.') (options, args) = parser.parse_args() if not options.input_vars: options.input_vars = os.path.join(options.etc, DEFAULT_INPUT_VARS) if not options.builder_dir: options.builder_dir = os.path.join(options.etc, DEFAULT_BUILDER_DIR) if not options.hosts: options.hosts = os.path.join(options.etc, DEFAULT_HOSTS) if not options.swift_ring_builder_consumes: options.swift_ring_builder_consumes = os.path.join( options.etc, DEFAULT_SWIFT_RING_BUILDER_CONSUMES) if not options.ring_delta: options.ring_delta = os.path.join(options.etc, DEFAULT_RING_DELTA) if not options.osconfig: options.osconfig = os.path.join(options.etc, DEFAULT_OSCONFIG) # # Work out what we need to do. Validate arguments needed by an action # are present. # actions = [] if options.storage_policies: actions.append('input-from-model') actions.append('emit-storage-policies') if not options.region_name: print('Need --region-name') sys.exit(1) if options.make_delta: actions.append('init-delta') actions.append('input-from-model') actions.append('read-builder-dir') actions.append('open-osconfig-dir') actions.append('make-delta') actions.append('write-to-delta') if not options.ring_delta: print('Need --ring-delta file to write to') sys.exit(1) if not (options.input_vars and options.hosts and options.osconfig and options.swift_ring_builder_consumes): print('Need --input-vars, --hosts and --osconfig and' ' --swift-ring-builder-consumes inputs') sys.exit(1) if not options.builder_dir: print('Need --builder-dir option') sys.exit(1) if options.fmt not in ['yaml', 'json']: print('Invalid value for --format') if options.report: actions.append('init-delta') actions.append('read-from-delta') actions.append('report') if not options.ring_delta: print('Need --ring-delta file as input') sys.exit(1) if options.detail not in ['summary', 'full']: print('Invalid value for --detail') sys.exit(1) if options.rebalance: actions.append('init-delta') actions.append('open-builder-dir') actions.append('read-from-delta') actions.append('rebalance') if not options.ring_delta: print('Need --ring-delta file as input') sys.exit(1) if options.fmt not in ['yaml', 'json']: print('Invalid value for --format') if len(actions) == 0: print('Missing an option to perform some action') sys.exit(1) if options.storage_policies and (options.make_delta or options.rebalance or options.report): print('Do not mix --storage-policies with other actions') sys.exit(1) if options.report and (options.make_delta or options.rebalance or options.storage_policies): print('Do not mix --report with other actions') sys.exit(1) # # Perform actions # if 'init-delta' in actions: delta = RingDelta() if 'input-from-model' in actions: try: input_model_fd = open(options.input_vars, 'r') hosts_fd = open(options.hosts, 'r') consumes_fd = open(options.swift_ring_builder_consumes, 'r') except IOError as err: print('ERROR: %s' % err) sys.exit(1) try: input_vars = safe_load(input_model_fd) consumes_model = safe_load(consumes_fd) except scanner.ScannerError as err: print('ERROR in %s: %s' % (options.input_vars, err)) sys.exit(err) try: input_model = InputModel(config=input_vars, hosts_fd=hosts_fd, consumes=consumes_model) ring_model = RingSpecifications(model=input_vars) except SwiftModelException as err: sys.exit(err) if 'emit-storage-policies' in actions: obj_ring_policies = ring_model.get_storage_policies( options.region_name) print('%s' % safe_dump(obj_ring_policies, default_flow_style=False)) # Unlike the following actions, we exit after this action sys.exit(0) if 'open-builder-dir' or 'read-builder-dir' in actions: try: read_write = False if 'read-builder-dir' in actions: read_write = True rings = RingBuilder(options.builder_dir, True) except IOError as err: print('ERROR: %s' % err) sys.exit(1) if 'open-osconfig-dir' in actions: drive_configurations = osconfig_load(options.osconfig) if 'make-delta' in actions: try: generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) except SwiftModelException as err: print('ERROR: %s' % err) sys.exit(1) if 'write-to-delta' in actions: if options.ring_delta == '-': write_to_file_fd = sys.stdout else: write_to_file_fd = open(options.ring_delta, 'w') delta.write_to_file(write_to_file_fd, options.fmt) if 'read-from-delta' in actions: if options.ring_delta == '-': print('--ring-delta=- is invalid (read from stdin not supported)') sys.exit(1) try: delta = RingDelta() read_from_delta_fd = open(options.ring_delta, 'r') delta.read_from_file(read_from_delta_fd, options.fmt) except IOError as err: print('ERROR: %s' % err) sys.exit(1) if 'report' in actions: print(delta.get_report(options.detail)) if 'rebalance' in actions: rebalance(delta, rings, options.dry_run)