Ejemplo n.º 1
0
 def test_override_replica_count_on_upgrade(self):
     options = DummyInputOptions()
     config_paths = CloudMultiSite(options)
     input_model = ServersModel('standard', 'ccp',
                                config=safe_load(standard_input_model),
                                consumes_model=standard_swf_rng_consumes)
     ring_model = RingSpecifications('standard', 'ccp',
                                     model=safe_load(standard_input_model))
     num_devices = input_model.get_num_devices('account')
     # Change the model to have too high a replica count
     account_ring_model = ring_model.get_control_plane_rings(
         'standard', 'ccp').get_ringspec('account')
     account_ring_model['replication_policy']['replica_count'] = \
         num_devices + 1
     # The fake pre-mitaka rings also have a high replica count
     rings = FakeRingBuilder(self.builder_dir,
                             ['account', 'container', 'object-0'],
                             replica_count=num_devices + 1)
     drive_configurations = dummy_osconfig_load(
         standard_drive_configurations)
     delta = RingDelta()
     try:
         # stop_on_warnings is True in unit tests
         generate_delta(config_paths, input_model, ring_model, rings,
                        drive_configurations, options, delta)
         self.assertTrue(False, msg='should not get here')
     except SwiftModelException:
         options.dry_run = True
         cmds = rebalance(delta, rings, options)
         self.assertTrue('account.builder'
                         ' set_replicas %s' % num_devices in
                         ' '.join(cmds))
Ejemplo n.º 2
0
    def test_missmatched_region_names(self):
        options = DummyInputOptions()
        input_model = InputModel(config=safe_load(padawan_input_model),
                                 hosts_fd=padawan_net_hosts,
                                 consumes=padawan_swf_rng_consumes)
        ring_model = RingSpecifications(model=safe_load(padawan_input_model))
        rings = RingBuilder(self.builder_dir, False)
        drive_configurations = dummy_osconfig_load(
            padawan_drive_configurations)
        delta = RingDelta()

        # Change region_name in rings
        for region_rings in ring_model.keystone_ring_specifications:
            region_rings.region_name = 'junk'
        self.assertRaises(SwiftModelException, generate_delta, input_model,
                          ring_model, rings, drive_configurations, options,
                          delta)
        # Put back original
        ring_model = RingSpecifications(model=safe_load(padawan_input_model))

        # Add a region - but not used
        keystone_region_rings = KeystoneRegionRings(self)
        keystone_region_rings.region_name = 'unusedrregion'
        ring_model.keystone_ring_specifications.append(keystone_region_rings)
        # Does not stop
        options.stop_on_warnings = False
        generate_delta(input_model, ring_model, rings, drive_configurations,
                       options, delta)
        # Stops on warnings
        options.stop_on_warnings = True
        self.assertRaises(SwiftModelException, generate_delta, input_model,
                          ring_model, rings, drive_configurations, options,
                          delta)
Ejemplo n.º 3
0
 def test_override_replica_count_on_install(self):
     options = DummyInputOptions()
     config_paths = CloudMultiSite(options)
     input_model = ServersModel('standard', 'ccp',
                                config=safe_load(standard_input_model),
                                consumes_model=standard_swf_rng_consumes)
     ring_model = RingSpecifications('standard', 'ccp',
                                     model=safe_load(standard_input_model))
     num_devices = input_model.get_num_devices('account')
     # Change the model to have too high a replica count
     account_ring_model = ring_model.get_control_plane_rings(
         'standard', 'ccp').get_ringspec('account')
     account_ring_model['replication_policy']['replica_count'] = \
         num_devices + 1
     rings = RingBuilder(self.builder_dir, False)
     drive_configurations = dummy_osconfig_load(
         standard_drive_configurations)
     delta = RingDelta()
     options.stop_on_warnings = False
     generate_delta(config_paths, input_model, ring_model, rings,
                    drive_configurations, options, delta)
     options.dry_run = True
     cmds = rebalance(delta, rings, options)
     self.assertTrue('account.builder'
                     ' create 17 %s 24' % float(num_devices) in
                     ' '.join(cmds))
Ejemplo n.º 4
0
 def test_missing_ring_specification(self):
     options = DummyInputOptions()
     config_paths = CloudMultiSite(options)
     input_model = ServersModel('standard', 'ccp',
                                config=safe_load(standard_input_model),
                                consumes_model=standard_swf_rng_consumes)
     # Change the disk model so a bad ring name is referenced
     for server in input_model.servers:
         disk_model = server.get('disk_model')
         for device_group in disk_model.get('device_groups', []):
             consumer = device_group.get('consumer')
             if consumer and consumer.get('name', 'other') == 'swift':
                 attrs = consumer.get('attrs')
                 attrs.get('rings').append({'name': 'object-99'})
     ring_model = RingSpecifications('standard', 'ccp',
                                     model=None)
     ring_model.load_configuration('standard', 'ccp',
                                   safe_load(standard_configuration_data))
     rings = RingBuilder(self.builder_dir, False)
     drive_configurations = dummy_osconfig_load(
         standard_drive_configurations)
     delta = RingDelta()
     try:
         generate_delta(config_paths, input_model, ring_model, rings,
                        drive_configurations, options, delta)
         self.assertTrue(False, msg='should not get here')
     except SwiftModelException as err:
         self.assertTrue('Cannot proceed' in str(err))
Ejemplo n.º 5
0
 def test_opaque_rack_ids(self):
     ring_model = RingSpecifications(model=safe_load(ringspec_region_zones))
     r, z = ring_model.get_region_zone('region1', 'account', 21)
     self.assertEquals((r, z), (2, -1))
     r, z = ring_model.get_region_zone('region1', 'account', '21')
     self.assertEquals((r, z), (2, -1))
     r, z = ring_model.get_region_zone('region2', 'account', '51')
     self.assertEquals((r, z), (5, 6))
Ejemplo n.º 6
0
 def test_null_region_zones(self):
     scenarios = [
         ('my_cloud', 'my_control_plane', 'account', ['any'], -1, None),
         ('my_cloud', 'my_control_plane', 'container', ['any'], -1, -1),
     ]
     ring_model = RingSpecifications('my_cloud',
                                     'my_control_plane',
                                     model=safe_load(ringspec_null_zones))
     for scenario in scenarios:
         cl, cp, rng, rck, rv, rz = scenario
         r, z = ring_model.get_control_plane_rings(cl, cp).get_region_zone(
             rng, rck)
         self.assertEquals((cl, cp, rng, rck, r, z),
                           (cl, cp, rng, rck, rv, rz))
Ejemplo n.º 7
0
    def test_noop(self):
        options = DummyInputOptions()
        input_model = InputModel(config=safe_load(padawan_input_model),
                                 hosts_fd=padawan_net_hosts,
                                 consumes=padawan_swf_rng_consumes)
        ring_model = RingSpecifications(model=safe_load(padawan_input_model))
        rings = FakeRingBuilder(
            self.builder_dir, 'regionone',
            ['account', 'container', 'object-0', 'object-1'], 3.0)
        drive_configurations = dummy_osconfig_load(
            padawan_drive_configurations)
        delta = RingDelta()
        generate_delta(input_model, ring_model, rings, drive_configurations,
                       options, delta)
        # Load the fake builder rings with the delta i.e., make it look as
        # though we had just done a rebalance() using input model
        rings.load_fake_ring_data(delta)

        # make a new delta and rebalance
        delta = RingDelta()
        generate_delta(input_model, ring_model, rings, drive_configurations,
                       options, delta)
        cmds = rebalance(delta, rings, True)
        # account, container, object-0:
        #     3 x rebalance (only) (which is point of this test)
        # object-1 has
        #     1 x set replica count
        #     2 x set weights
        #     1 x rebalance
        # total: 7
        self.assertTrue(len(cmds) == 7)
Ejemplo n.º 8
0
    def test_set_weight_large_step(self):
        options = DummyInputOptions()
        config_paths = CloudMultiSite(options)
        input_model = ServersModel('standard', 'ccp',
                                   config=safe_load(standard_input_model),
                                   consumes_model=standard_swf_rng_consumes)
        ring_model = RingSpecifications('standard', 'ccp',
                                        model=safe_load(standard_input_model))
        rings = FakeRingBuilder(self.builder_dir,
                                ['account', 'container', 'object-0'],
                                3.0)
        drive_configurations = dummy_osconfig_load(
            standard_drive_configurations)
        options.weight_step = 999999
        delta = RingDelta()
        generate_delta(config_paths, input_model, ring_model, rings,
                       drive_configurations, options, delta)
        # Load the fake builder rings with the delta i.e., make it look as
        # though we had just done a rebalance() using input model
        rings.load_fake_ring_data(delta)

        # Change the weights to a small value
        rings.fake_set_weights(1.0)

        # make a new delta and rebalance
        delta = RingDelta()
        generate_delta(config_paths, input_model, ring_model, rings,
                       drive_configurations, options, delta)
        options.dry_run = True
        cmds = rebalance(delta, rings, options)
        self.assertTrue('account.builder  set_weight'
                        ' 192.168.245.2/disk0 18.63' in ' '.join(cmds))
Ejemplo n.º 9
0
 def test_simple_ringspecs(self):
     ring_model = RingSpecifications('my_cloud',
                                     'my_control_plane',
                                     model=safe_load(ringspec_simple))
     self.assertEquals(
         len(ring_model.control_planes[('my_cloud',
                                        'my_control_plane')].rings), 4)
Ejemplo n.º 10
0
    def test_set_weight_no_step(self):
        options = DummyInputOptions()
        input_model = InputModel(config=safe_load(padawan_input_model),
                                 hosts_fd=padawan_net_hosts,
                                 consumes=padawan_swf_rng_consumes)
        ring_model = RingSpecifications(model=safe_load(padawan_input_model))
        rings = FakeRingBuilder(self.builder_dir, 'regionone',
                                ['account', 'container', 'object-0'], 3.0)
        drive_configurations = dummy_osconfig_load(
            padawan_drive_configurations)
        delta = RingDelta()
        generate_delta(input_model, ring_model, rings, drive_configurations,
                       options, delta)
        # Load the fake builder rings with the delta i.e., make it look as
        # though we had just done a rebalance() using input model
        rings.load_fake_ring_data(delta)

        # Change the weights to a small value
        rings.fake_set_weights(1.0)

        # make a new delta and rebalance
        delta = RingDelta()
        generate_delta(input_model, ring_model, rings, drive_configurations,
                       options, delta)
        cmds = rebalance(delta, rings, True)
        self.assertTrue('account.builder  set_weight'
                        ' 192.168.245.2/disk0 18.63' in ' '.join(cmds))
Ejemplo n.º 11
0
 def test_change_replica_count(self):
     options = DummyInputOptions()
     input_model = InputModel(config=safe_load(padawan_input_model),
                              hosts_fd=padawan_net_hosts,
                              consumes=padawan_swf_rng_consumes)
     ring_model = RingSpecifications(model=safe_load(padawan_input_model))
     rings = FakeRingBuilder(self.builder_dir, 'regionone', ['container'],
                             4.0)
     drive_configurations = dummy_osconfig_load(
         padawan_drive_configurations)
     delta = RingDelta()
     generate_delta(input_model, ring_model, rings, drive_configurations,
                    options, delta)
     cmds = rebalance(delta, rings, True)
     # Fake container ring has replica-count of 4.0, check that we
     # change it to match the model (3.0)
     self.assertTrue(
         verb_ringname_args_in_cmds('set_replicas', 'container.builder',
                                    ['3.0'], cmds))
     # Validate we don't attempt to re-create container
     self.assertTrue(not verb_ringname_args_in_cmds(
         'create', 'container.builder', None, cmds))
     # Validate other rings are created
     self.assertTrue(
         verb_ringname_args_in_cmds('create', 'account.builder',
                                    ['17', '3.0', '24'], cmds))
     self.assertTrue(
         verb_ringname_args_in_cmds('create', 'object-0.builder',
                                    ['17', '3.0', '24'], cmds))
Ejemplo n.º 12
0
    def test_set_weight_removing(self):
        options = DummyInputOptions()
        config_paths = CloudMultiSite(options)
        input_model = ServersModel('standard', 'ccp',
                                   config=safe_load(standard_input_model),
                                   consumes_model=standard_swf_rng_consumes)
        ring_model = RingSpecifications('standard', 'ccp',
                                        model=safe_load(standard_input_model))
        rings = FakeRingBuilder(self.builder_dir,
                                ['account', 'container', 'object-0'],
                                3.0)
        drive_configurations = dummy_osconfig_load(
            standard_drive_configurations)
        delta = RingDelta()
        generate_delta(config_paths, input_model, ring_model, rings,
                       drive_configurations, options, delta)
        # Load the fake builder rings with the delta i.e., make it look as
        # though we had just done a rebalance() using input model
        rings.load_fake_ring_data(delta)

        # Set standard-ccp-c1-m3 (92.168.245.2) to removing
        for server in input_model.servers:
            if server.get('ardana_ansible_host') == 'standard-ccp-c1-m3':
                server['pass_through'] = {'swift': {'remove': True}}
        self.assertTrue(input_model.server_removing('standard-ccp-c1-m3'))

        # make a new delta and rebalance
        delta = RingDelta()
        generate_delta(config_paths, input_model, ring_model, rings,
                       drive_configurations, options, delta)
        options.dry_run = True
        cmds = rebalance(delta, rings, options)
        self.assertTrue('account.builder  remove'
                        ' 192.168.245.2/disk0' in ' '.join(cmds))
Ejemplo n.º 13
0
    def test_set_weight_with_step(self):
        options = DummyInputOptions()
        config_paths = CloudMultiSite(options)
        input_model = ServersModel('standard', 'ccp',
                                   config=safe_load(standard_input_model),
                                   consumes_model=standard_swf_rng_consumes)
        ring_model = RingSpecifications('standard', 'ccp',
                                        model=safe_load(standard_input_model))
        rings = FakeRingBuilder(self.builder_dir,
                                ['account', 'container', 'object-0'],
                                3.0)
        drive_configurations = dummy_osconfig_load(
            standard_drive_configurations)
        delta = RingDelta()
        generate_delta(config_paths, input_model, ring_model, rings,
                       drive_configurations, options, delta)
        # Load the fake builder rings with the delta i.e., make it look as
        # though we had just done a rebalance() using input model
        rings.load_fake_ring_data(delta)

        # Change the weights to a small value
        rings.fake_set_weights(1.0)

        # This make delta has a weight_step
        options.weight_step = '10.0'
        delta = RingDelta()
        generate_delta(config_paths, input_model, ring_model, rings,
                       drive_configurations, options, delta)
        options.dry_run = True
        cmds = rebalance(delta, rings, options)
        self.assertTrue('account.builder  set_weight'
                        ' 192.168.245.2/disk0 11.00' in ' '.join(cmds))

        # Go through another cycle -- update as though last step built
        # the rings - use small step
        rings = FakeRingBuilder(self.builder_dir,
                                ['account', 'container', 'object-0'],
                                3.0)
        rings.load_fake_ring_data(delta)
        options.weight_step = '1.0'
        delta = RingDelta()
        generate_delta(config_paths, input_model, ring_model, rings,
                       drive_configurations, options, delta)
        cmds = rebalance(delta, rings, options)
        self.assertTrue('account.builder  set_weight'
                        ' 192.168.245.2/disk0 12.00' in ' '.join(cmds))

        # Go through another cycle -- the step is large enough that final
        # target weight is reached
        rings = FakeRingBuilder(self.builder_dir,
                                ['account', 'container', 'object-0'],
                                3.0)
        rings.load_fake_ring_data(delta)
        options.weight_step = '10.0'
        delta = RingDelta()
        generate_delta(config_paths, input_model, ring_model, rings,
                       drive_configurations, options, delta)
        cmds = rebalance(delta, rings, options)
        self.assertTrue('account.builder  set_weight'
                        ' 192.168.245.2/disk0 18.63' in ' '.join(cmds))
Ejemplo n.º 14
0
 def test_add_servers(self):
     # This test uses same process as test_build_rings() above -- so it
     # appears as though all servers are being added to an existing
     # system. With --weight-step=10, the resulting weights are limited
     # to 10.0
     options = DummyInputOptions()
     config_paths = CloudMultiSite(options)
     input_model = ServersModel('standard', 'ccp',
                                config=safe_load(standard_input_model),
                                consumes_model=standard_swf_rng_consumes)
     ring_model = RingSpecifications('standard', 'ccp',
                                     model=safe_load(standard_input_model))
     rings = RingBuilder(self.builder_dir, False)
     drive_configurations = dummy_osconfig_load(
         standard_drive_configurations)
     # Set limit to weight
     options.weight_step = '10'
     delta = RingDelta()
     generate_delta(config_paths, input_model, ring_model, rings,
                    drive_configurations, options, delta)
     options.dry_run = True
     cmds = rebalance(delta, rings, options)
     self.assertTrue('--device disk0'
                     ' --meta standard-ccp-c1-m3:disk0:/dev/sdc'
                     ' --weight 10.00' in ' '.join(cmds))
     self.assertTrue('--device lvm0 --meta'
                     ' standard-ccp-c1-m2:lvm0:/dev/ardana-vg/LV_SWFAC'
                     ' --weight 10.00' in ' '.join(cmds))
Ejemplo n.º 15
0
 def test_add_servers(self):
     # This test uses same process as test_build_rings() above -- so it
     # appears as though all servers are being added to an existing
     # system. With --weight-step=10, the resulting weights are limited
     # to 10.0
     options = DummyInputOptions()
     input_model = InputModel(config=safe_load(padawan_input_model),
                              hosts_fd=padawan_net_hosts,
                              consumes=padawan_swf_rng_consumes)
     ring_model = RingSpecifications(model=safe_load(padawan_input_model))
     rings = RingBuilder(self.builder_dir, False)
     drive_configurations = dummy_osconfig_load(
         padawan_drive_configurations)
     # Set limit to weight
     options.weight_step = '10'
     delta = RingDelta()
     generate_delta(input_model, ring_model, rings, drive_configurations,
                    options, delta)
     cmds = rebalance(delta, rings, True)
     self.assertTrue('--device disk0'
                     ' --meta padawan-ccp-c1-m3:disk0:/dev/sdc'
                     ' --weight 10.00' in ' '.join(cmds))
     self.assertTrue('--device lvm0 --meta'
                     ' padawan-ccp-c1-m2:lvm0:/dev/hlm-vg/LV_SWFAC'
                     ' --weight 10.00' in ' '.join(cmds))
Ejemplo n.º 16
0
    def test_set_weight_down(self):

        options = DummyInputOptions()
        input_model = InputModel(config=safe_load(padawan_input_model),
                                 hosts_fd=padawan_net_hosts,
                                 consumes=padawan_swf_rng_consumes)
        ring_model = RingSpecifications(model=safe_load(padawan_input_model))
        rings = FakeRingBuilder(self.builder_dir, 'regionone',
                                ['account', 'container', 'object-0'], 3.0)
        drive_configurations = dummy_osconfig_load(
            padawan_drive_configurations)
        delta = RingDelta()
        generate_delta(input_model, ring_model, rings, drive_configurations,
                       options, delta)
        # Load the fake builder rings with the delta i.e., make it look as
        # though we had just done a rebalance() using input model
        rings.load_fake_ring_data(delta)

        # Change the weights to a large value
        rings.fake_set_weights(30.0)

        # This make delta has a weight_step
        options.weight_step = '10.0'
        delta = RingDelta()
        generate_delta(input_model, ring_model, rings, drive_configurations,
                       options, delta)
        cmds = rebalance(delta, rings, True)
        self.assertTrue('account.builder  set_weight'
                        ' 192.168.245.2/disk0 20.00' in ' '.join(cmds))

        # Go through another cycle -- update as though last step built
        # the rings - use small step
        rings = FakeRingBuilder(self.builder_dir, 'regionone',
                                ['account', 'container', 'object-0'], 3.0)
        rings.load_fake_ring_data(delta)
        options.weight_step = '1.0'
        delta = RingDelta()
        generate_delta(input_model, ring_model, rings, drive_configurations,
                       options, delta)
        cmds = rebalance(delta, rings, True)
        self.assertTrue('account.builder  set_weight'
                        ' 192.168.245.2/disk0 19.00' in ' '.join(cmds))

        # Go through another cycle -- the step is large enough that final
        # target weight is reached
        rings = FakeRingBuilder(self.builder_dir, 'regionone',
                                ['account', 'container', 'object-0'], 3.0)
        rings.load_fake_ring_data(delta)
        options.weight_step = '10.0'
        delta = RingDelta()
        generate_delta(input_model, ring_model, rings, drive_configurations,
                       options, delta)
        cmds = rebalance(delta, rings, True)
        self.assertTrue('account.builder  set_weight'
                        ' 192.168.245.2/disk0 18.63' in ' '.join(cmds))
Ejemplo n.º 17
0
 def test_region_zones(self):
     scenarios = [
         ('my_cloud', 'my_control_plane', 'container', ['sg21'], 2, -1),
         ('my_cloud', 'my_control_plane', 'container',
          ['other21', 'sg21', 'other22'], 2, -1),
         ('my_cloud', 'my_control_plane', 'object-0', ['sg31'], 3, -1),
         ('my_cloud', 'my_control_plane', 'object-0',
          ['other31', 'sg31', 'other32'], 3, -1),
         ('my_cloud', 'my_control_plane', 'object-0', ['junk1',
                                                       'junk2'], None, -1),
         ('my_cloud', 'my_control_plane', 'container', [], None, -1),
     ]
     ring_model = RingSpecifications('my_cloud',
                                     'my_control_plane',
                                     model=safe_load(ringspec_region_zones))
     for scenario in scenarios:
         cl, cp, rng, rck, rv, rz = scenario
         r, z = ring_model.get_control_plane_rings(cl, cp).get_region_zone(
             rng, rck)
         self.assertEquals((cl, cp, rng, rck, r, z),
                           (cl, cp, rng, rck, rv, rz))
Ejemplo n.º 18
0
 def test_rack_ids(self):
     scenarios = [
         ('region1', 'container', 21, 2, -1),
         ('region1', 'object-0', 31, 3, -1),
         ('region2', 'account', 'four3', 4, 6),
         ('region2', 'account', 52, 5, 6),
         ('region2', 'object-0', 'four1', 4, 7),
         ('region2', 'object-0', 53, 5, 8),
         ('region2', 'container', 91, 9, None),
         ('region2', 'container', 101, None, 10),
         ('region2', 'bad-ring', 52, 5, None),
         ('bad-region', 'account', 52, None, None),
         ('region1', 'container', None, None, -1),
         ('region2', 'container', None, None, None),
         ('region2', 'object-0', None, None, None),
     ]
     ring_model = RingSpecifications(model=safe_load(ringspec_region_zones))
     for scenario in scenarios:
         rg, rng, rck, rv, rz = scenario
         r, z = ring_model.get_region_zone(rg, rng, rck)
         self.assertEquals((rg, rng, rck, r, z), (rg, rng, rck, rv, rz))
Ejemplo n.º 19
0
 def test_build_rings(self):
     options = DummyInputOptions()
     config_paths = CloudMultiSite(options)
     input_model = ServersModel('standard', 'ccp',
                                config=safe_load(standard_input_model),
                                consumes_model=standard_swf_rng_consumes)
     # Use rings from configuration-data object
     ring_model = RingSpecifications('standard', 'ccp',
                                     model=None)
     ring_model.load_configuration('standard', 'ccp',
                                   safe_load(standard_configuration_data))
     rings = RingBuilder(self.builder_dir, False)
     drive_configurations = dummy_osconfig_load(
         standard_drive_configurations)
     delta = RingDelta()
     generate_delta(config_paths, input_model, ring_model, rings,
                    drive_configurations, options, delta)
     options.dry_run = True
     # Validate pretend has no effect since all rings are new
     options.pretend_min_part_hours_passed = True
     cmds = rebalance(delta, rings, options)
     assert_cmds_are_same(self, expected_cmds, cmds)
Ejemplo n.º 20
0
 def test_build_rings(self):
     options = DummyInputOptions()
     input_model = InputModel(config=safe_load(padawan_input_model),
                              hosts_fd=padawan_net_hosts,
                              consumes=padawan_swf_rng_consumes)
     ring_model = RingSpecifications(model=safe_load(padawan_input_model))
     rings = RingBuilder(self.builder_dir, False)
     drive_configurations = dummy_osconfig_load(
         padawan_drive_configurations)
     delta = RingDelta()
     generate_delta(input_model, ring_model, rings, drive_configurations,
                    options, delta)
     cmds = rebalance(delta, rings, True)
     assert_cmds_are_same(self, expected_cmds, cmds)
Ejemplo n.º 21
0
 def test_replication_replica_count(self):
     ring_model = RingSpecifications(model=safe_load(ringspec_simple))
     self.assertEquals(
         ring_model.get_ringspec('region1', 'account').replica_count, 1.0)
     self.assertEquals(
         ring_model.get_ringspec('region1', 'container').replica_count, 2.0)
     self.assertEquals(
         ring_model.get_ringspec('region1', 'object-0').replica_count, 3.0)
     self.assertEquals(
         ring_model.get_ringspec('region1', 'object-1').replica_count, 14.0)
Ejemplo n.º 22
0
 def test_build_limit_ring(self):
     options = DummyInputOptions()
     config_paths = CloudMultiSite(options)
     input_model = ServersModel('standard', 'ccp',
                                config=safe_load(standard_input_model),
                                consumes_model=standard_swf_rng_consumes)
     ring_model = RingSpecifications('standard', 'ccp',
                                     model=safe_load(standard_input_model))
     rings = RingBuilder(self.builder_dir, False)
     drive_configurations = dummy_osconfig_load(
         standard_drive_configurations)
     delta = RingDelta()
     generate_delta(config_paths, input_model, ring_model, rings,
                    drive_configurations, options, delta)
     options.dry_run = True
     options.limit_ring = 'object-1'
     cmds = rebalance(delta, rings, options)
     self.assertEqual(len(cmds), 4)
Ejemplo n.º 23
0
 def test_change_replica_count_min_part_hours(self):
     options = DummyInputOptions()
     config_paths = CloudMultiSite(options)
     input_model = ServersModel('standard', 'ccp',
                                config=safe_load(standard_input_model),
                                consumes_model=standard_swf_rng_consumes)
     ring_model = RingSpecifications('standard', 'ccp',
                                     model=safe_load(standard_input_model))
     rings = FakeRingBuilder(self.builder_dir,
                             ['container'], replica_count=4.0,
                             min_part_hours=6)
     drive_configurations = dummy_osconfig_load(
         standard_drive_configurations)
     delta = RingDelta()
     generate_delta(config_paths, input_model, ring_model, rings,
                    drive_configurations, options, delta)
     options.dry_run = True
     cmds = rebalance(delta, rings, options)
     # Fake container ring has replica-count of 4.0, check that we
     # change it to match the model (3.0)
     self.assertTrue(verb_ringname_args_in_cmds('set_replicas',
                                                'container.builder',
                                                ['3.0'], cmds))
     # Fake container ring has min-part-hours of 6, check that we
     # change it to match the model (24)
     self.assertTrue(verb_ringname_args_in_cmds('set_min_part_hours',
                                                'container.builder',
                                                ['24'], cmds))
     # Validate we don't attempt to re-create container
     self.assertTrue(not verb_ringname_args_in_cmds('create',
                                                    'container.builder',
                                                    None, cmds))
     # Validate other rings are created
     self.assertTrue(verb_ringname_args_in_cmds('create',
                                                'account.builder',
                                                ['17', '3.0', '24'], cmds))
     self.assertTrue(verb_ringname_args_in_cmds('create',
                                                'object-0.builder',
                                                ['17', '3.0', '24'], cmds))
Ejemplo n.º 24
0
    def test_noop(self):
        options = DummyInputOptions()
        config_paths = CloudMultiSite(options)
        input_model = ServersModel('standard', 'ccp',
                                   config=safe_load(standard_input_model),
                                   consumes_model=standard_swf_rng_consumes)
        ring_model = RingSpecifications('standard', 'ccp',
                                        model=safe_load(standard_input_model))
        rings = FakeRingBuilder(self.builder_dir,
                                ['account', 'container', 'object-0',
                                 'object-1'],
                                3.0)
        drive_configurations = dummy_osconfig_load(
            standard_drive_configurations)
        delta = RingDelta()
        generate_delta(config_paths, input_model, ring_model, rings,
                       drive_configurations, options, delta)
        # Load the fake builder rings with the delta i.e., make it look as
        # though we had just done a rebalance() using input model
        rings.load_fake_ring_data(delta)

        # make a new delta and rebalance
        delta = RingDelta()
        generate_delta(config_paths, input_model, ring_model, rings,
                       drive_configurations, options, delta)
        options.dry_run = True
        cmds = rebalance(delta, rings, options)
        # account, container, object-0:
        #     3 x rebalance (only) (which is point of this test)
        # object-1 has
        #     1 x set replica count
        #     1 x set min part hours
        #     2 x set weights
        #     1 x rebalance
        # total: 8
        self.assertTrue(len(cmds) == 8)
Ejemplo n.º 25
0
 def test_replication_replica_count(self):
     ring_model = RingSpecifications('my_cloud',
                                     'my_control_plane',
                                     model=safe_load(ringspec_simple))
     self.assertEquals(
         ring_model.get_control_plane_rings(
             'my_cloud',
             'my_control_plane').get_ringspec('account').replica_count, 1.0)
     self.assertEquals(
         ring_model.get_control_plane_rings(
             'my_cloud',
             'my_control_plane').get_ringspec('container').replica_count,
         2.0)
     self.assertEquals(
         ring_model.get_control_plane_rings(
             'my_cloud',
             'my_control_plane').get_ringspec('object-0').replica_count,
         3.0)
     self.assertEquals(
         ring_model.get_control_plane_rings(
             'my_cloud',
             'my_control_plane').get_ringspec('object-1').replica_count,
         14.0)
Ejemplo n.º 26
0
 def test_load_region_zones(self):
     ring_model = RingSpecifications(model=safe_load(ringspec_region_zones))
     self.assertEquals(
         ring_model.keystone_ring_specifications[0].region_name, 'region1')
     self.assertEquals(
         ring_model.keystone_ring_specifications[1].region_name, 'region2')
Ejemplo n.º 27
0
def main():
    parser = OptionParser(usage=usage)
    parser.add_option('--etc',
                      dest='etc',
                      default=DEFAULT_ETC,
                      help='Overrides /etc/swiftlm (for testing)')
    parser.add_option('--cloud',
                      dest='cloud',
                      default=None,
                      help='The name of the cloud')
    parser.add_option('--control-plane',
                      dest='control_plane',
                      default=None,
                      help='The name of the control plane')
    parser.add_option('--ring-delta',
                      dest='ring_delta',
                      default=None,
                      help='Name of ring-delta file (as output or input'
                      ' A value of "-" (on output means to write'
                      ' to stdout')
    parser.add_option('--format',
                      dest='fmt',
                      default='yaml',
                      help='One of yaml or json.'
                      ' When used with --ring-delta, specifies the'
                      ' format of the file.')
    parser.add_option('--detail',
                      dest='detail',
                      default='summary',
                      help='Level of detail to use with --report.'
                      ' Use summary or full')
    parser.add_option('--report',
                      dest='report',
                      default=False,
                      action="store_true",
                      help='Explain what the ring delta represents.'
                      ' Optionally use --detail.')
    parser.add_option('--dry-run',
                      dest='dry_run',
                      default=False,
                      action="store_true",
                      help='Show the proposed swift-ring-builder commands')
    parser.add_option('--pretend-min-part-hours-passed',
                      dest='pretend_min_part_hours_passed',
                      default=False,
                      action="store_true",
                      help='Executes the pretend_min_part_hours_passed command'
                      ' on each ring before running rebalance.'
                      ' Use with caution.')
    parser.add_option('--make-delta',
                      dest='make_delta',
                      default=False,
                      action="store_true",
                      help='Make a ring delta file')
    parser.add_option('--rebalance',
                      dest='rebalance',
                      default=False,
                      action="store_true",
                      help='Build (or rebalance) rings')
    parser.add_option('--limit-ring',
                      dest='limit_ring',
                      default=None,
                      help='Limits actions to given ring')
    parser.add_option('--size-to-weight',
                      dest='size_to_weight',
                      default=float(1024 * 1024 * 1024),
                      help='Conversion factor for size to weight. Default is'
                      ' 1GB is weight of 1 (a 4Tb drive would be assigned'
                      ' a weight of 4096')
    parser.add_option('--weight-step',
                      dest='weight_step',
                      default=None,
                      help='When set, weights are changed by at most this'
                      ' value. Overrides value in ring specification.')
    parser.add_option('--allow-partitions',
                      dest='allow_partitions',
                      default=False,
                      action='store_true',
                      help='Allow devices to be assigned to partitions.'
                      ' Default is to use a full disk drive.')
    parser.add_option('--stop-on-warnings',
                      dest='stop_on_warnings',
                      default=False,
                      action='store_true',
                      help='Used with --make-delta. Exit with error if there'
                      ' are model missmatch warnings.'
                      ' Default is to only exit with error for errors.')
    parser.add_option('--unittest',
                      dest='unittest',
                      default=False,
                      action='store_true',
                      help='Set by unittests. Never set on command line.')
    (options, args) = parser.parse_args()

    if not (options.cloud and options.control_plane):
        sys.exit('Must specify both --cloud and --control_plane')

    sites = CloudMultiSite(options)
    my_cloud = sites.my_cloud
    my_control_plane = sites.my_control_plane
    my_config = sites.path(my_cloud, my_control_plane)

    #
    # Work out what we need to do. Validate arguments needed by an action
    # are present.
    #
    actions = []
    if options.make_delta:
        actions.append('init-delta')
        actions.append('input-from-model')
        actions.append('read-builder-dir')
        actions.append('open-osconfig-dir')
        actions.append('make-delta')
        actions.append('write-to-delta')

        if options.fmt not in ['yaml', 'json']:
            print('Invalid value for --format')

    if options.report:
        actions.append('init-delta')
        actions.append('read-from-delta')
        actions.append('report')

        if options.detail not in ['summary', 'full']:
            sys.exit('Invalid value for --detail')

    if options.rebalance:
        actions.append('init-delta')
        actions.append('open-builder-dir')
        actions.append('read-from-delta')
        actions.append('rebalance')

        if options.fmt not in ['yaml', 'json']:
            print('Invalid value for --format')

    if len(actions) == 0:
        sys.exit('Missing an option to perform some action')
    if options.report and (options.make_delta or options.rebalance):
        sys.exit('Do not mix --report with other actions')

    #
    # Perform actions
    #
    if 'init-delta' in actions:
        delta = RingDelta()

    if 'input-from-model' in actions:
        servers_model = ServersModel('unused', 'unused')
        consumes = Consumes()
        ring_model = RingSpecifications(my_cloud, my_control_plane)
        for cloud, control_plane in sites.control_planes():
            config = sites.path(cloud, control_plane)
            input_model_fd = None
            try:
                input_model_fd = open(config.get('input-model'), 'r')
            except IOError as err:
                pass  # File may not exist since its a legacy item
            try:
                cp_server_fd = None
                cp_server_fd = open(config.get('control_plane_servers'), 'r')
            except IOError as err:
                sys.exit('Error on control_plane_server.yml: %s' % err)
            try:
                control_plane_servers = None
                if cp_server_fd:
                    control_plane_servers = safe_load(cp_server_fd)
            except scanner.ScannerError as err:
                sys.exit('ERROR reading/parsing: %s' % err)
            try:
                consumes_fd = open(config.get('swift_ring_builder_consumes'),
                                   'r')
            except IOError as err:
                sys.exit('ERROR: %s' % err)
            try:
                input_vars = {'global': {}}
                if input_model_fd:
                    input_vars = safe_load(input_model_fd)
                consumes_model = safe_load(consumes_fd)
            except scanner.ScannerError as err:
                sys.exit('ERROR reading/parsing: %s' % err)
            try:
                if control_plane_servers:
                    servers = control_plane_servers.get(
                        'control_plane_servers')
                elif input_vars.get('global').get('all_servers'):
                    servers = input_vars.get('global').get('all_servers')
                else:
                    sys.exit('No servers found in control plane')
                servers_model.add_servers(cloud, control_plane, servers)
                consumes.load_model(consumes_model)
            except SwiftModelException as err:
                sys.exit(err)
        servers_model.register_consumes(consumes)

        try:
            config_data_fd = open(my_config.get('configuration_data'), 'r')
            config_data = safe_load(config_data_fd)
        except (IOError, scanner.ScannerError) as err:
            sys.exit('Rings should be in configuration-data.'
                     ' Using old configuration processor?')
        try:
            rings_loaded = False
            if input_vars.get('global').get('all_ring_specifications'):
                # Model contains Ardana old-style rings
                ring_model = RingSpecifications(my_cloud,
                                                my_control_plane,
                                                model=input_vars)
                rings_loaded = True
            if config_data and config_data.get(
                    'control_plane_rings',
                    config_data.get('control-plane-rings')):
                # Model contains new-style rings --- use instead
                ring_model.load_configuration(my_cloud, my_control_plane,
                                              config_data)
                rings_loaded = True
            if not rings_loaded:
                sys.exit('No ring specifications in input model')
        except SwiftModelException as err:
            sys.exit(err)

    if 'open-builder-dir' or 'read-builder-dir' in actions:
        try:
            read_rings = False
            if 'read-builder-dir' in actions:
                read_rings = True
            rings = RingBuilder(my_config.get('builder_dir'),
                                read_rings=read_rings)
        except IOError as err:
            sys.exit('ERROR: %s' % err)

    if 'open-osconfig-dir' in actions:
        drive_configurations = osconfig_load(sites)

    if 'make-delta' in actions:
        try:
            generate_delta(sites, servers_model, ring_model, rings,
                           drive_configurations, options, delta)
        except SwiftModelException as err:
            sys.exit('ERROR: %s' % err)

    if 'write-to-delta' in actions:
        if my_config.get('ring-delta') == '-':
            write_to_file_fd = sys.stdout
        else:
            write_to_file_fd = open(my_config.get('ring-delta'), 'w')
        delta.write_to_file(write_to_file_fd, options.fmt)

    if 'read-from-delta' in actions:
        if my_config.get('ring-delta') == '-':
            sys.exit('--ring-delta- is invalid (read from stdin'
                     'not supported)')
        try:
            delta = RingDelta()
            read_from_delta_fd = open(my_config.get('ring-delta'), 'r')
            delta.read_from_file(read_from_delta_fd, options.fmt)
        except IOError as err:
            sys.exit('ERROR: %s' % err)

    if 'report' in actions:
        print(delta.get_report(options))

    if 'rebalance' in actions:
        rebalance(delta, rings, options)
Ejemplo n.º 28
0
def main():
    parser = OptionParser(usage=usage)
    parser.add_option('--etc',
                      dest='etc',
                      default=DEFAULT_ETC,
                      help='Overrides /etc/swiftlm (for testing)')
    parser.add_option('--input-vars',
                      dest='input_vars',
                      default=None,
                      help='The name of yaml containing vars')
    parser.add_option('--builder-dir',
                      dest='builder_dir',
                      default=None,
                      help='Builder file directory')
    parser.add_option('--osconfig',
                      dest='osconfig',
                      default=None,
                      help='Directory where drive size data files exist.'
                      ' If omitted, the correct weights cannot'
                      ' be assigned to drives.')
    parser.add_option('--hosts',
                      dest='hosts',
                      default=None,
                      help='Hosts file (/etc/hosts or ./net/hosts.hf)')
    parser.add_option('--swift-ring-builder-consumes',
                      dest='swift_ring_builder_consumes',
                      default=None,
                      help='File containing the SWF_RNG variable')
    parser.add_option('--ring-delta',
                      dest='ring_delta',
                      default=None,
                      help='Name of ring-delta file (as output or input'
                      ' A value of "-" (on output means to write'
                      ' to stdout')
    parser.add_option('--format',
                      dest='fmt',
                      default='yaml',
                      help='One of yaml or json.'
                      ' When used with --ring-delta, specifies the'
                      ' format of the file.')
    parser.add_option('--detail',
                      dest='detail',
                      default='summary',
                      help='Level of detail to use with --report.'
                      ' Use summary or full')
    parser.add_option('--report',
                      dest='report',
                      default=False,
                      action="store_true",
                      help='Explain what the ring delta represents.'
                      ' Optionally use --detail.')
    parser.add_option('--dry-run',
                      dest='dry_run',
                      default=False,
                      action="store_true",
                      help='Show the proposed swift-ring-builder commands')
    parser.add_option('--make-delta',
                      dest='make_delta',
                      default=False,
                      action="store_true",
                      help='Make a ring delta file')
    parser.add_option('--rebalance',
                      dest='rebalance',
                      default=False,
                      action="store_true",
                      help='Build (or rebalance) rings')
    parser.add_option('--storage-policies',
                      dest='storage_policies',
                      default=False,
                      action="store_true",
                      help='Extract storage policy data.'
                      ' Use this to register the storage_policies'
                      ' variable.')
    parser.add_option('--region-name',
                      dest='region_name',
                      default=None,
                      help='Region name for use with --storage-policies')
    parser.add_option('--size-to-weight',
                      dest='size_to_weight',
                      default=float(1024 * 1024 * 1024),
                      help='Conversion factor for size to weight. Default is'
                      ' 1GB is weight of 1 (a 4Tb drive would be assigned'
                      ' a weight of 4096')
    parser.add_option('--weight-step',
                      dest='weight_step',
                      default=None,
                      help='When set, weights are changed by at most this'
                      ' value. Overrides value in ring specification.')
    parser.add_option('--allow-partitions',
                      dest='allow_partitions',
                      default=False,
                      action='store_true',
                      help='Allow devices to be assigned to partitions.'
                      ' Default is to use a full disk drive.')
    parser.add_option('--stop-on-warnings',
                      dest='stop_on_warnings',
                      default=False,
                      action='store_true',
                      help='Used with --make-delta. Exit with error if there'
                      ' are model missmatch warnings.'
                      ' Default is to only exit with error for errors.')
    (options, args) = parser.parse_args()

    if not options.input_vars:
        options.input_vars = os.path.join(options.etc, DEFAULT_INPUT_VARS)
    if not options.builder_dir:
        options.builder_dir = os.path.join(options.etc, DEFAULT_BUILDER_DIR)
    if not options.hosts:
        options.hosts = os.path.join(options.etc, DEFAULT_HOSTS)
    if not options.swift_ring_builder_consumes:
        options.swift_ring_builder_consumes = os.path.join(
            options.etc, DEFAULT_SWIFT_RING_BUILDER_CONSUMES)
    if not options.ring_delta:
        options.ring_delta = os.path.join(options.etc, DEFAULT_RING_DELTA)
    if not options.osconfig:
        options.osconfig = os.path.join(options.etc, DEFAULT_OSCONFIG)

    #
    # Work out what we need to do. Validate arguments needed by an action
    # are present.
    #
    actions = []
    if options.storage_policies:
        actions.append('input-from-model')
        actions.append('emit-storage-policies')

        if not options.region_name:
            print('Need --region-name')
            sys.exit(1)

    if options.make_delta:
        actions.append('init-delta')
        actions.append('input-from-model')
        actions.append('read-builder-dir')
        actions.append('open-osconfig-dir')
        actions.append('make-delta')
        actions.append('write-to-delta')

        if not options.ring_delta:
            print('Need --ring-delta file to write to')
            sys.exit(1)
        if not (options.input_vars and options.hosts and options.osconfig
                and options.swift_ring_builder_consumes):
            print('Need --input-vars, --hosts and --osconfig and'
                  ' --swift-ring-builder-consumes inputs')
            sys.exit(1)
        if not options.builder_dir:
            print('Need --builder-dir option')
            sys.exit(1)
        if options.fmt not in ['yaml', 'json']:
            print('Invalid value for --format')

    if options.report:
        actions.append('init-delta')
        actions.append('read-from-delta')
        actions.append('report')

        if not options.ring_delta:
            print('Need --ring-delta file as input')
            sys.exit(1)
        if options.detail not in ['summary', 'full']:
            print('Invalid value for --detail')
            sys.exit(1)

    if options.rebalance:
        actions.append('init-delta')
        actions.append('open-builder-dir')
        actions.append('read-from-delta')
        actions.append('rebalance')

        if not options.ring_delta:
            print('Need --ring-delta file as input')
            sys.exit(1)
        if options.fmt not in ['yaml', 'json']:
            print('Invalid value for --format')

    if len(actions) == 0:
        print('Missing an option to perform some action')
        sys.exit(1)
    if options.storage_policies and (options.make_delta or options.rebalance
                                     or options.report):
        print('Do not mix --storage-policies with other actions')
        sys.exit(1)
    if options.report and (options.make_delta or options.rebalance
                           or options.storage_policies):
        print('Do not mix --report with other actions')
        sys.exit(1)

    #
    # Perform actions
    #
    if 'init-delta' in actions:
        delta = RingDelta()

    if 'input-from-model' in actions:
        try:
            input_model_fd = open(options.input_vars, 'r')
            hosts_fd = open(options.hosts, 'r')
            consumes_fd = open(options.swift_ring_builder_consumes, 'r')
        except IOError as err:
            print('ERROR: %s' % err)
            sys.exit(1)
        try:
            input_vars = safe_load(input_model_fd)
            consumes_model = safe_load(consumes_fd)
        except scanner.ScannerError as err:
            print('ERROR in %s: %s' % (options.input_vars, err))
            sys.exit(err)
        try:
            input_model = InputModel(config=input_vars,
                                     hosts_fd=hosts_fd,
                                     consumes=consumes_model)
            ring_model = RingSpecifications(model=input_vars)
        except SwiftModelException as err:
            sys.exit(err)

    if 'emit-storage-policies' in actions:
        obj_ring_policies = ring_model.get_storage_policies(
            options.region_name)
        print('%s' % safe_dump(obj_ring_policies, default_flow_style=False))

        # Unlike the following actions, we exit after this action
        sys.exit(0)

    if 'open-builder-dir' or 'read-builder-dir' in actions:
        try:
            read_write = False
            if 'read-builder-dir' in actions:
                read_write = True
            rings = RingBuilder(options.builder_dir, True)
        except IOError as err:
            print('ERROR: %s' % err)
            sys.exit(1)

    if 'open-osconfig-dir' in actions:
        drive_configurations = osconfig_load(options.osconfig)

    if 'make-delta' in actions:
        try:
            generate_delta(input_model, ring_model, rings,
                           drive_configurations, options, delta)
        except SwiftModelException as err:
            print('ERROR: %s' % err)
            sys.exit(1)

    if 'write-to-delta' in actions:
        if options.ring_delta == '-':
            write_to_file_fd = sys.stdout
        else:
            write_to_file_fd = open(options.ring_delta, 'w')
        delta.write_to_file(write_to_file_fd, options.fmt)

    if 'read-from-delta' in actions:
        if options.ring_delta == '-':
            print('--ring-delta=- is invalid (read from stdin not supported)')
            sys.exit(1)
        try:
            delta = RingDelta()
            read_from_delta_fd = open(options.ring_delta, 'r')
            delta.read_from_file(read_from_delta_fd, options.fmt)
        except IOError as err:
            print('ERROR: %s' % err)
            sys.exit(1)

    if 'report' in actions:
        print(delta.get_report(options.detail))

    if 'rebalance' in actions:
        rebalance(delta, rings, options.dry_run)
Ejemplo n.º 29
0
 def test_storage_policies(self):
     ring_model = RingSpecifications(model=safe_load(ringspec_simple))
     sp = ring_model.get_storage_policies('region1')
     self.assertEquals(sp['storage_policies'][0]['policy']['index'], 0)
Ejemplo n.º 30
0
def main():
    parser = OptionParser(usage=usage)
    parser.add_option('--etc', dest='etc',
                      default=DEFAULT_ETC,
                      help='Overrides /etc/swiftlm (for testing)')
    parser.add_option('--input-vars', dest='input_vars', default=None,
                      help='The name of yaml containing vars')
    parser.add_option('--builder-dir', dest='builder_dir', default=None,
                      help='Builder file directory')
    parser.add_option('--osconfig', dest='osconfig', default=None,
                      help='Directory where drive size data files exist.'
                           ' If omitted, the correct weights cannot'
                           ' be assigned to drives.')
    parser.add_option('--hosts', dest='hosts', default=None,
                      help='Hosts file (/etc/hosts or ./net/hosts.hf)')
    parser.add_option('--swift-ring-builder-consumes',
                      dest='swift_ring_builder_consumes',
                      default=None,
                      help='File containing the SWF_RNG variable')
    parser.add_option('--ring-delta', dest='ring_delta', default=None,
                      help='Name of ring-delta file (as output or input'
                           ' A value of "-" (on output means to write'
                           ' to stdout')
    parser.add_option('--format', dest='fmt', default='yaml',
                      help='One of yaml or json.'
                           ' When used with --ring-delta, specifies the'
                           ' format of the file.')
    parser.add_option('--detail', dest='detail', default='summary',
                      help='Level of detail to use with --report.'
                           ' Use summary or full')
    parser.add_option('--report', dest='report', default=False,
                      action="store_true",
                      help='Explain what the ring delta represents.'
                           ' Optionally use --detail.')
    parser.add_option('--dry-run', dest='dry_run', default=False,
                      action="store_true",
                      help='Show the proposed swift-ring-builder commands')
    parser.add_option('--make-delta', dest='make_delta', default=False,
                      action="store_true",
                      help='Make a ring delta file')
    parser.add_option('--rebalance', dest='rebalance', default=False,
                      action="store_true",
                      help='Build (or rebalance) rings')
    parser.add_option('--storage-policies', dest='storage_policies',
                      default=False,
                      action="store_true",
                      help='Extract storage policy data.'
                           ' Use this to register the storage_policies'
                           ' variable.')
    parser.add_option('--region-name', dest='region_name', default=None,
                      help='Region name for use with --storage-policies')
    parser.add_option('--size-to-weight', dest='size_to_weight',
                      default=float(1024 * 1024 * 1024),
                      help='Conversion factor for size to weight. Default is'
                           ' 1GB is weight of 1 (a 4Tb drive would be assigned'
                           ' a weight of 4096')
    parser.add_option('--weight-step', dest='weight_step',
                      default=None,
                      help='When set, weights are changed by at most this'
                           ' value. Overrides value in ring specification.')
    parser.add_option('--allow-partitions', dest='allow_partitions',
                      default=False, action='store_true',
                      help='Allow devices to be assigned to partitions.'
                           ' Default is to use a full disk drive.')
    parser.add_option('--stop-on-warnings', dest='stop_on_warnings',
                      default=False, action='store_true',
                      help='Used with --make-delta. Exit with error if there'
                           ' are model missmatch warnings.'
                           ' Default is to only exit with error for errors.')
    (options, args) = parser.parse_args()

    if not options.input_vars:
        options.input_vars = os.path.join(options.etc, DEFAULT_INPUT_VARS)
    if not options.builder_dir:
        options.builder_dir = os.path.join(options.etc, DEFAULT_BUILDER_DIR)
    if not options.hosts:
        options.hosts = os.path.join(options.etc, DEFAULT_HOSTS)
    if not options.swift_ring_builder_consumes:
        options.swift_ring_builder_consumes = os.path.join(
            options.etc, DEFAULT_SWIFT_RING_BUILDER_CONSUMES)
    if not options.ring_delta:
        options.ring_delta = os.path.join(options.etc, DEFAULT_RING_DELTA)
    if not options.osconfig:
        options.osconfig = os.path.join(options.etc, DEFAULT_OSCONFIG)

    #
    # Work out what we need to do. Validate arguments needed by an action
    # are present.
    #
    actions = []
    if options.storage_policies:
        actions.append('input-from-model')
        actions.append('emit-storage-policies')

        if not options.region_name:
            print('Need --region-name')
            sys.exit(1)

    if options.make_delta:
        actions.append('init-delta')
        actions.append('input-from-model')
        actions.append('read-builder-dir')
        actions.append('open-osconfig-dir')
        actions.append('make-delta')
        actions.append('write-to-delta')

        if not options.ring_delta:
            print('Need --ring-delta file to write to')
            sys.exit(1)
        if not (options.input_vars and options.hosts and
                options.osconfig and options.swift_ring_builder_consumes):
            print('Need --input-vars, --hosts and --osconfig and'
                  ' --swift-ring-builder-consumes inputs')
            sys.exit(1)
        if not options.builder_dir:
            print('Need --builder-dir option')
            sys.exit(1)
        if options.fmt not in ['yaml', 'json']:
            print('Invalid value for --format')

    if options.report:
        actions.append('init-delta')
        actions.append('read-from-delta')
        actions.append('report')

        if not options.ring_delta:
            print('Need --ring-delta file as input')
            sys.exit(1)
        if options.detail not in ['summary', 'full']:
            print('Invalid value for --detail')
            sys.exit(1)

    if options.rebalance:
        actions.append('init-delta')
        actions.append('open-builder-dir')
        actions.append('read-from-delta')
        actions.append('rebalance')

        if not options.ring_delta:
            print('Need --ring-delta file as input')
            sys.exit(1)
        if options.fmt not in ['yaml', 'json']:
            print('Invalid value for --format')

    if len(actions) == 0:
        print('Missing an option to perform some action')
        sys.exit(1)
    if options.storage_policies and (options.make_delta or
                                     options.rebalance or
                                     options.report):
        print('Do not mix --storage-policies with other actions')
        sys.exit(1)
    if options.report and (options.make_delta or
                           options.rebalance or
                           options.storage_policies):
        print('Do not mix --report with other actions')
        sys.exit(1)

    #
    # Perform actions
    #
    if 'init-delta' in actions:
        delta = RingDelta()

    if 'input-from-model' in actions:
        try:
            input_model_fd = open(options.input_vars, 'r')
            hosts_fd = open(options.hosts, 'r')
            consumes_fd = open(options.swift_ring_builder_consumes, 'r')
        except IOError as err:
            print('ERROR: %s' % err)
            sys.exit(1)
        try:
            input_vars = safe_load(input_model_fd)
            consumes_model = safe_load(consumes_fd)
        except scanner.ScannerError as err:
            print('ERROR in %s: %s' % (options.input_vars, err))
            sys.exit(err)
        try:
            input_model = InputModel(config=input_vars, hosts_fd=hosts_fd,
                                     consumes=consumes_model)
            ring_model = RingSpecifications(model=input_vars)
        except SwiftModelException as err:
            sys.exit(err)

    if 'emit-storage-policies' in actions:
        obj_ring_policies = ring_model.get_storage_policies(
            options.region_name)
        print('%s' % safe_dump(obj_ring_policies, default_flow_style=False))

        # Unlike the following actions, we exit after this action
        sys.exit(0)

    if 'open-builder-dir' or 'read-builder-dir' in actions:
        try:
            read_write = False
            if 'read-builder-dir' in actions:
                read_write = True
            rings = RingBuilder(options.builder_dir, True)
        except IOError as err:
            print('ERROR: %s' % err)
            sys.exit(1)

    if 'open-osconfig-dir' in actions:
        drive_configurations = osconfig_load(options.osconfig)

    if 'make-delta' in actions:
        try:
            generate_delta(input_model, ring_model, rings,
                           drive_configurations, options, delta)
        except SwiftModelException as err:
            print('ERROR: %s' % err)
            sys.exit(1)

    if 'write-to-delta' in actions:
        if options.ring_delta == '-':
            write_to_file_fd = sys.stdout
        else:
            write_to_file_fd = open(options.ring_delta, 'w')
        delta.write_to_file(write_to_file_fd, options.fmt)

    if 'read-from-delta' in actions:
        if options.ring_delta == '-':
            print('--ring-delta=- is invalid (read from stdin not supported)')
            sys.exit(1)
        try:
            delta = RingDelta()
            read_from_delta_fd = open(options.ring_delta, 'r')
            delta.read_from_file(read_from_delta_fd, options.fmt)
        except IOError as err:
            print('ERROR: %s' % err)
            sys.exit(1)

    if 'report' in actions:
        print(delta.get_report(options.detail))

    if 'rebalance' in actions:
        rebalance(delta, rings, options.dry_run)
Ejemplo n.º 31
0
 def test_simple_ringspecs(self):
     ring_model = RingSpecifications(model=safe_load(ringspec_simple))
     self.assertEquals(
         ring_model.keystone_ring_specifications[0].region_name, 'region1')