def test_set_weight_removing(self):
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
ring_model = RingSpecifications('standard', 'ccp',
model=safe_load(standard_input_model))
rings = FakeRingBuilder(self.builder_dir,
['account', 'container', 'object-0'],
3.0)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
# Load the fake builder rings with the delta i.e., make it look as
# though we had just done a rebalance() using input model
rings.load_fake_ring_data(delta)
# Set standard-ccp-c1-m3 (92.168.245.2) to removing
for server in input_model.servers:
if server.get('ardana_ansible_host') == 'standard-ccp-c1-m3':
server['pass_through'] = {'swift': {'remove': True}}
self.assertTrue(input_model.server_removing('standard-ccp-c1-m3'))
# make a new delta and rebalance
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
options.dry_run = True
cmds = rebalance(delta, rings, options)
self.assertTrue('account.builder remove'
' 192.168.245.2/disk0' in ' '.join(cmds))
def test_set_weight_with_step(self):
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
ring_model = RingSpecifications('standard', 'ccp',
model=safe_load(standard_input_model))
rings = FakeRingBuilder(self.builder_dir,
['account', 'container', 'object-0'],
3.0)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
# Load the fake builder rings with the delta i.e., make it look as
# though we had just done a rebalance() using input model
rings.load_fake_ring_data(delta)
# Change the weights to a small value
rings.fake_set_weights(1.0)
# This make delta has a weight_step
options.weight_step = '10.0'
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
options.dry_run = True
cmds = rebalance(delta, rings, options)
self.assertTrue('account.builder set_weight'
' 192.168.245.2/disk0 11.00' in ' '.join(cmds))
# Go through another cycle -- update as though last step built
# the rings - use small step
rings = FakeRingBuilder(self.builder_dir,
['account', 'container', 'object-0'],
3.0)
rings.load_fake_ring_data(delta)
options.weight_step = '1.0'
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
cmds = rebalance(delta, rings, options)
self.assertTrue('account.builder set_weight'
' 192.168.245.2/disk0 12.00' in ' '.join(cmds))
# Go through another cycle -- the step is large enough that final
# target weight is reached
rings = FakeRingBuilder(self.builder_dir,
['account', 'container', 'object-0'],
3.0)
rings.load_fake_ring_data(delta)
options.weight_step = '10.0'
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
cmds = rebalance(delta, rings, options)
self.assertTrue('account.builder set_weight'
' 192.168.245.2/disk0 18.63' in ' '.join(cmds))
def test_set_weight_no_step(self):
options = DummyInputOptions()
input_model = InputModel(config=safe_load(padawan_input_model),
hosts_fd=padawan_net_hosts,
consumes=padawan_swf_rng_consumes)
ring_model = RingSpecifications(model=safe_load(padawan_input_model))
rings = FakeRingBuilder(self.builder_dir, 'regionone',
['account', 'container', 'object-0'], 3.0)
drive_configurations = dummy_osconfig_load(
padawan_drive_configurations)
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
# Load the fake builder rings with the delta i.e., make it look as
# though we had just done a rebalance() using input model
rings.load_fake_ring_data(delta)
# Change the weights to a small value
rings.fake_set_weights(1.0)
# make a new delta and rebalance
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
cmds = rebalance(delta, rings, True)
self.assertTrue('account.builder set_weight'
' 192.168.245.2/disk0 18.63' in ' '.join(cmds))
def test_set_weight_large_step(self):
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
ring_model = RingSpecifications('standard', 'ccp',
model=safe_load(standard_input_model))
rings = FakeRingBuilder(self.builder_dir,
['account', 'container', 'object-0'],
3.0)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
options.weight_step = 999999
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
# Load the fake builder rings with the delta i.e., make it look as
# though we had just done a rebalance() using input model
rings.load_fake_ring_data(delta)
# Change the weights to a small value
rings.fake_set_weights(1.0)
# make a new delta and rebalance
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
options.dry_run = True
cmds = rebalance(delta, rings, options)
self.assertTrue('account.builder set_weight'
' 192.168.245.2/disk0 18.63' in ' '.join(cmds))
def test_noop(self):
options = DummyInputOptions()
input_model = InputModel(config=safe_load(padawan_input_model),
hosts_fd=padawan_net_hosts,
consumes=padawan_swf_rng_consumes)
ring_model = RingSpecifications(model=safe_load(padawan_input_model))
rings = FakeRingBuilder(
self.builder_dir, 'regionone',
['account', 'container', 'object-0', 'object-1'], 3.0)
drive_configurations = dummy_osconfig_load(
padawan_drive_configurations)
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
# Load the fake builder rings with the delta i.e., make it look as
# though we had just done a rebalance() using input model
rings.load_fake_ring_data(delta)
# make a new delta and rebalance
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
cmds = rebalance(delta, rings, True)
# account, container, object-0:
# 3 x rebalance (only) (which is point of this test)
# object-1 has
# 1 x set replica count
# 2 x set weights
# 1 x rebalance
# total: 7
self.assertTrue(len(cmds) == 7)
def test_set_weight_down(self):
options = DummyInputOptions()
input_model = InputModel(config=safe_load(padawan_input_model),
hosts_fd=padawan_net_hosts,
consumes=padawan_swf_rng_consumes)
ring_model = RingSpecifications(model=safe_load(padawan_input_model))
rings = FakeRingBuilder(self.builder_dir, 'regionone',
['account', 'container', 'object-0'], 3.0)
drive_configurations = dummy_osconfig_load(
padawan_drive_configurations)
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
# Load the fake builder rings with the delta i.e., make it look as
# though we had just done a rebalance() using input model
rings.load_fake_ring_data(delta)
# Change the weights to a large value
rings.fake_set_weights(30.0)
# This make delta has a weight_step
options.weight_step = '10.0'
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
cmds = rebalance(delta, rings, True)
self.assertTrue('account.builder set_weight'
' 192.168.245.2/disk0 20.00' in ' '.join(cmds))
# Go through another cycle -- update as though last step built
# the rings - use small step
rings = FakeRingBuilder(self.builder_dir, 'regionone',
['account', 'container', 'object-0'], 3.0)
rings.load_fake_ring_data(delta)
options.weight_step = '1.0'
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
cmds = rebalance(delta, rings, True)
self.assertTrue('account.builder set_weight'
' 192.168.245.2/disk0 19.00' in ' '.join(cmds))
# Go through another cycle -- the step is large enough that final
# target weight is reached
rings = FakeRingBuilder(self.builder_dir, 'regionone',
['account', 'container', 'object-0'], 3.0)
rings.load_fake_ring_data(delta)
options.weight_step = '10.0'
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
cmds = rebalance(delta, rings, True)
self.assertTrue('account.builder set_weight'
' 192.168.245.2/disk0 18.63' in ' '.join(cmds))
def test_override_replica_count_on_install(self):
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
ring_model = RingSpecifications('standard', 'ccp',
model=safe_load(standard_input_model))
num_devices = input_model.get_num_devices('account')
# Change the model to have too high a replica count
account_ring_model = ring_model.get_control_plane_rings(
'standard', 'ccp').get_ringspec('account')
account_ring_model['replication_policy']['replica_count'] = \
num_devices + 1
rings = RingBuilder(self.builder_dir, False)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
delta = RingDelta()
options.stop_on_warnings = False
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
options.dry_run = True
cmds = rebalance(delta, rings, options)
self.assertTrue('account.builder'
' create 17 %s 24' % float(num_devices) in
' '.join(cmds))
def test_override_replica_count_on_upgrade(self):
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
ring_model = RingSpecifications('standard', 'ccp',
model=safe_load(standard_input_model))
num_devices = input_model.get_num_devices('account')
# Change the model to have too high a replica count
account_ring_model = ring_model.get_control_plane_rings(
'standard', 'ccp').get_ringspec('account')
account_ring_model['replication_policy']['replica_count'] = \
num_devices + 1
# The fake pre-mitaka rings also have a high replica count
rings = FakeRingBuilder(self.builder_dir,
['account', 'container', 'object-0'],
replica_count=num_devices + 1)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
delta = RingDelta()
try:
# stop_on_warnings is True in unit tests
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
self.assertTrue(False, msg='should not get here')
except SwiftModelException:
options.dry_run = True
cmds = rebalance(delta, rings, options)
self.assertTrue('account.builder'
' set_replicas %s' % num_devices in
' '.join(cmds))
def test_add_servers(self):
# This test uses same process as test_build_rings() above -- so it
# appears as though all servers are being added to an existing
# system. With --weight-step=10, the resulting weights are limited
# to 10.0
options = DummyInputOptions()
input_model = InputModel(config=safe_load(padawan_input_model),
hosts_fd=padawan_net_hosts,
consumes=padawan_swf_rng_consumes)
ring_model = RingSpecifications(model=safe_load(padawan_input_model))
rings = RingBuilder(self.builder_dir, False)
drive_configurations = dummy_osconfig_load(
padawan_drive_configurations)
# Set limit to weight
options.weight_step = '10'
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
cmds = rebalance(delta, rings, True)
self.assertTrue('--device disk0'
' --meta padawan-ccp-c1-m3:disk0:/dev/sdc'
' --weight 10.00' in ' '.join(cmds))
self.assertTrue('--device lvm0 --meta'
' padawan-ccp-c1-m2:lvm0:/dev/hlm-vg/LV_SWFAC'
' --weight 10.00' in ' '.join(cmds))
def test_change_replica_count(self):
options = DummyInputOptions()
input_model = InputModel(config=safe_load(padawan_input_model),
hosts_fd=padawan_net_hosts,
consumes=padawan_swf_rng_consumes)
ring_model = RingSpecifications(model=safe_load(padawan_input_model))
rings = FakeRingBuilder(self.builder_dir, 'regionone', ['container'],
4.0)
drive_configurations = dummy_osconfig_load(
padawan_drive_configurations)
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
cmds = rebalance(delta, rings, True)
# Fake container ring has replica-count of 4.0, check that we
# change it to match the model (3.0)
self.assertTrue(
verb_ringname_args_in_cmds('set_replicas', 'container.builder',
['3.0'], cmds))
# Validate we don't attempt to re-create container
self.assertTrue(not verb_ringname_args_in_cmds(
'create', 'container.builder', None, cmds))
# Validate other rings are created
self.assertTrue(
verb_ringname_args_in_cmds('create', 'account.builder',
['17', '3.0', '24'], cmds))
self.assertTrue(
verb_ringname_args_in_cmds('create', 'object-0.builder',
['17', '3.0', '24'], cmds))
def test_missmatched_region_names(self):
options = DummyInputOptions()
input_model = InputModel(config=safe_load(padawan_input_model),
hosts_fd=padawan_net_hosts,
consumes=padawan_swf_rng_consumes)
ring_model = RingSpecifications(model=safe_load(padawan_input_model))
rings = RingBuilder(self.builder_dir, False)
drive_configurations = dummy_osconfig_load(
padawan_drive_configurations)
delta = RingDelta()
# Change region_name in rings
for region_rings in ring_model.keystone_ring_specifications:
region_rings.region_name = 'junk'
self.assertRaises(SwiftModelException, generate_delta, input_model,
ring_model, rings, drive_configurations, options,
delta)
# Put back original
ring_model = RingSpecifications(model=safe_load(padawan_input_model))
# Add a region - but not used
keystone_region_rings = KeystoneRegionRings(self)
keystone_region_rings.region_name = 'unusedrregion'
ring_model.keystone_ring_specifications.append(keystone_region_rings)
# Does not stop
options.stop_on_warnings = False
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
# Stops on warnings
options.stop_on_warnings = True
self.assertRaises(SwiftModelException, generate_delta, input_model,
ring_model, rings, drive_configurations, options,
delta)
def test_add_servers(self):
# This test uses same process as test_build_rings() above -- so it
# appears as though all servers are being added to an existing
# system. With --weight-step=10, the resulting weights are limited
# to 10.0
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
ring_model = RingSpecifications('standard', 'ccp',
model=safe_load(standard_input_model))
rings = RingBuilder(self.builder_dir, False)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
# Set limit to weight
options.weight_step = '10'
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
options.dry_run = True
cmds = rebalance(delta, rings, options)
self.assertTrue('--device disk0'
' --meta standard-ccp-c1-m3:disk0:/dev/sdc'
' --weight 10.00' in ' '.join(cmds))
self.assertTrue('--device lvm0 --meta'
' standard-ccp-c1-m2:lvm0:/dev/ardana-vg/LV_SWFAC'
' --weight 10.00' in ' '.join(cmds))
def test_missing_ring_specification(self):
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
# Change the disk model so a bad ring name is referenced
for server in input_model.servers:
disk_model = server.get('disk_model')
for device_group in disk_model.get('device_groups', []):
consumer = device_group.get('consumer')
if consumer and consumer.get('name', 'other') == 'swift':
attrs = consumer.get('attrs')
attrs.get('rings').append({'name': 'object-99'})
ring_model = RingSpecifications('standard', 'ccp',
model=None)
ring_model.load_configuration('standard', 'ccp',
safe_load(standard_configuration_data))
rings = RingBuilder(self.builder_dir, False)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
delta = RingDelta()
try:
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
self.assertTrue(False, msg='should not get here')
except SwiftModelException as err:
self.assertTrue('Cannot proceed' in str(err))
def test_build_rings(self):
options = DummyInputOptions()
input_model = InputModel(config=safe_load(padawan_input_model),
hosts_fd=padawan_net_hosts,
consumes=padawan_swf_rng_consumes)
ring_model = RingSpecifications(model=safe_load(padawan_input_model))
rings = RingBuilder(self.builder_dir, False)
drive_configurations = dummy_osconfig_load(
padawan_drive_configurations)
delta = RingDelta()
generate_delta(input_model, ring_model, rings, drive_configurations,
options, delta)
cmds = rebalance(delta, rings, True)
assert_cmds_are_same(self, expected_cmds, cmds)
def test_noop(self):
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
ring_model = RingSpecifications('standard', 'ccp',
model=safe_load(standard_input_model))
rings = FakeRingBuilder(self.builder_dir,
['account', 'container', 'object-0',
'object-1'],
3.0)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
# Load the fake builder rings with the delta i.e., make it look as
# though we had just done a rebalance() using input model
rings.load_fake_ring_data(delta)
# make a new delta and rebalance
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
options.dry_run = True
cmds = rebalance(delta, rings, options)
# account, container, object-0:
# 3 x rebalance (only) (which is point of this test)
# object-1 has
# 1 x set replica count
# 1 x set min part hours
# 2 x set weights
# 1 x rebalance
# total: 8
self.assertTrue(len(cmds) == 8)
def test_build_limit_ring(self):
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
ring_model = RingSpecifications('standard', 'ccp',
model=safe_load(standard_input_model))
rings = RingBuilder(self.builder_dir, False)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
options.dry_run = True
options.limit_ring = 'object-1'
cmds = rebalance(delta, rings, options)
self.assertEqual(len(cmds), 4)
def test_build_rings(self):
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
# Use rings from configuration-data object
ring_model = RingSpecifications('standard', 'ccp',
model=None)
ring_model.load_configuration('standard', 'ccp',
safe_load(standard_configuration_data))
rings = RingBuilder(self.builder_dir, False)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
options.dry_run = True
# Validate pretend has no effect since all rings are new
options.pretend_min_part_hours_passed = True
cmds = rebalance(delta, rings, options)
assert_cmds_are_same(self, expected_cmds, cmds)
def test_change_replica_count_min_part_hours(self):
options = DummyInputOptions()
config_paths = CloudMultiSite(options)
input_model = ServersModel('standard', 'ccp',
config=safe_load(standard_input_model),
consumes_model=standard_swf_rng_consumes)
ring_model = RingSpecifications('standard', 'ccp',
model=safe_load(standard_input_model))
rings = FakeRingBuilder(self.builder_dir,
['container'], replica_count=4.0,
min_part_hours=6)
drive_configurations = dummy_osconfig_load(
standard_drive_configurations)
delta = RingDelta()
generate_delta(config_paths, input_model, ring_model, rings,
drive_configurations, options, delta)
options.dry_run = True
cmds = rebalance(delta, rings, options)
# Fake container ring has replica-count of 4.0, check that we
# change it to match the model (3.0)
self.assertTrue(verb_ringname_args_in_cmds('set_replicas',
'container.builder',
['3.0'], cmds))
# Fake container ring has min-part-hours of 6, check that we
# change it to match the model (24)
self.assertTrue(verb_ringname_args_in_cmds('set_min_part_hours',
'container.builder',
['24'], cmds))
# Validate we don't attempt to re-create container
self.assertTrue(not verb_ringname_args_in_cmds('create',
'container.builder',
None, cmds))
# Validate other rings are created
self.assertTrue(verb_ringname_args_in_cmds('create',
'account.builder',
['17', '3.0', '24'], cmds))
self.assertTrue(verb_ringname_args_in_cmds('create',
'object-0.builder',
['17', '3.0', '24'], cmds))
def main():
parser = OptionParser(usage=usage)
parser.add_option('--etc',
dest='etc',
default=DEFAULT_ETC,
help='Overrides /etc/swiftlm (for testing)')
parser.add_option('--input-vars',
dest='input_vars',
default=None,
help='The name of yaml containing vars')
parser.add_option('--builder-dir',
dest='builder_dir',
default=None,
help='Builder file directory')
parser.add_option('--osconfig',
dest='osconfig',
default=None,
help='Directory where drive size data files exist.'
' If omitted, the correct weights cannot'
' be assigned to drives.')
parser.add_option('--hosts',
dest='hosts',
default=None,
help='Hosts file (/etc/hosts or ./net/hosts.hf)')
parser.add_option('--swift-ring-builder-consumes',
dest='swift_ring_builder_consumes',
default=None,
help='File containing the SWF_RNG variable')
parser.add_option('--ring-delta',
dest='ring_delta',
default=None,
help='Name of ring-delta file (as output or input'
' A value of "-" (on output means to write'
' to stdout')
parser.add_option('--format',
dest='fmt',
default='yaml',
help='One of yaml or json.'
' When used with --ring-delta, specifies the'
' format of the file.')
parser.add_option('--detail',
dest='detail',
default='summary',
help='Level of detail to use with --report.'
' Use summary or full')
parser.add_option('--report',
dest='report',
default=False,
action="store_true",
help='Explain what the ring delta represents.'
' Optionally use --detail.')
parser.add_option('--dry-run',
dest='dry_run',
default=False,
action="store_true",
help='Show the proposed swift-ring-builder commands')
parser.add_option('--make-delta',
dest='make_delta',
default=False,
action="store_true",
help='Make a ring delta file')
parser.add_option('--rebalance',
dest='rebalance',
default=False,
action="store_true",
help='Build (or rebalance) rings')
parser.add_option('--storage-policies',
dest='storage_policies',
default=False,
action="store_true",
help='Extract storage policy data.'
' Use this to register the storage_policies'
' variable.')
parser.add_option('--region-name',
dest='region_name',
default=None,
help='Region name for use with --storage-policies')
parser.add_option('--size-to-weight',
dest='size_to_weight',
default=float(1024 * 1024 * 1024),
help='Conversion factor for size to weight. Default is'
' 1GB is weight of 1 (a 4Tb drive would be assigned'
' a weight of 4096')
parser.add_option('--weight-step',
dest='weight_step',
default=None,
help='When set, weights are changed by at most this'
' value. Overrides value in ring specification.')
parser.add_option('--allow-partitions',
dest='allow_partitions',
default=False,
action='store_true',
help='Allow devices to be assigned to partitions.'
' Default is to use a full disk drive.')
parser.add_option('--stop-on-warnings',
dest='stop_on_warnings',
default=False,
action='store_true',
help='Used with --make-delta. Exit with error if there'
' are model missmatch warnings.'
' Default is to only exit with error for errors.')
(options, args) = parser.parse_args()
if not options.input_vars:
options.input_vars = os.path.join(options.etc, DEFAULT_INPUT_VARS)
if not options.builder_dir:
options.builder_dir = os.path.join(options.etc, DEFAULT_BUILDER_DIR)
if not options.hosts:
options.hosts = os.path.join(options.etc, DEFAULT_HOSTS)
if not options.swift_ring_builder_consumes:
options.swift_ring_builder_consumes = os.path.join(
options.etc, DEFAULT_SWIFT_RING_BUILDER_CONSUMES)
if not options.ring_delta:
options.ring_delta = os.path.join(options.etc, DEFAULT_RING_DELTA)
if not options.osconfig:
options.osconfig = os.path.join(options.etc, DEFAULT_OSCONFIG)
#
# Work out what we need to do. Validate arguments needed by an action
# are present.
#
actions = []
if options.storage_policies:
actions.append('input-from-model')
actions.append('emit-storage-policies')
if not options.region_name:
print('Need --region-name')
sys.exit(1)
if options.make_delta:
actions.append('init-delta')
actions.append('input-from-model')
actions.append('read-builder-dir')
actions.append('open-osconfig-dir')
actions.append('make-delta')
actions.append('write-to-delta')
if not options.ring_delta:
print('Need --ring-delta file to write to')
sys.exit(1)
if not (options.input_vars and options.hosts and options.osconfig
and options.swift_ring_builder_consumes):
print('Need --input-vars, --hosts and --osconfig and'
' --swift-ring-builder-consumes inputs')
sys.exit(1)
if not options.builder_dir:
print('Need --builder-dir option')
sys.exit(1)
if options.fmt not in ['yaml', 'json']:
print('Invalid value for --format')
if options.report:
actions.append('init-delta')
actions.append('read-from-delta')
actions.append('report')
if not options.ring_delta:
print('Need --ring-delta file as input')
sys.exit(1)
if options.detail not in ['summary', 'full']:
print('Invalid value for --detail')
sys.exit(1)
if options.rebalance:
actions.append('init-delta')
actions.append('open-builder-dir')
actions.append('read-from-delta')
actions.append('rebalance')
if not options.ring_delta:
print('Need --ring-delta file as input')
sys.exit(1)
if options.fmt not in ['yaml', 'json']:
print('Invalid value for --format')
if len(actions) == 0:
print('Missing an option to perform some action')
sys.exit(1)
if options.storage_policies and (options.make_delta or options.rebalance
or options.report):
print('Do not mix --storage-policies with other actions')
sys.exit(1)
if options.report and (options.make_delta or options.rebalance
or options.storage_policies):
print('Do not mix --report with other actions')
sys.exit(1)
#
# Perform actions
#
if 'init-delta' in actions:
delta = RingDelta()
if 'input-from-model' in actions:
try:
input_model_fd = open(options.input_vars, 'r')
hosts_fd = open(options.hosts, 'r')
consumes_fd = open(options.swift_ring_builder_consumes, 'r')
except IOError as err:
print('ERROR: %s' % err)
sys.exit(1)
try:
input_vars = safe_load(input_model_fd)
consumes_model = safe_load(consumes_fd)
except scanner.ScannerError as err:
print('ERROR in %s: %s' % (options.input_vars, err))
sys.exit(err)
try:
input_model = InputModel(config=input_vars,
hosts_fd=hosts_fd,
consumes=consumes_model)
ring_model = RingSpecifications(model=input_vars)
except SwiftModelException as err:
sys.exit(err)
if 'emit-storage-policies' in actions:
obj_ring_policies = ring_model.get_storage_policies(
options.region_name)
print('%s' % safe_dump(obj_ring_policies, default_flow_style=False))
# Unlike the following actions, we exit after this action
sys.exit(0)
if 'open-builder-dir' or 'read-builder-dir' in actions:
try:
read_write = False
if 'read-builder-dir' in actions:
read_write = True
rings = RingBuilder(options.builder_dir, True)
except IOError as err:
print('ERROR: %s' % err)
sys.exit(1)
if 'open-osconfig-dir' in actions:
drive_configurations = osconfig_load(options.osconfig)
if 'make-delta' in actions:
try:
generate_delta(input_model, ring_model, rings,
drive_configurations, options, delta)
except SwiftModelException as err:
print('ERROR: %s' % err)
sys.exit(1)
if 'write-to-delta' in actions:
if options.ring_delta == '-':
write_to_file_fd = sys.stdout
else:
write_to_file_fd = open(options.ring_delta, 'w')
delta.write_to_file(write_to_file_fd, options.fmt)
if 'read-from-delta' in actions:
if options.ring_delta == '-':
print('--ring-delta=- is invalid (read from stdin not supported)')
sys.exit(1)
try:
delta = RingDelta()
read_from_delta_fd = open(options.ring_delta, 'r')
delta.read_from_file(read_from_delta_fd, options.fmt)
except IOError as err:
print('ERROR: %s' % err)
sys.exit(1)
if 'report' in actions:
print(delta.get_report(options.detail))
if 'rebalance' in actions:
rebalance(delta, rings, options.dry_run)
def main():
parser = OptionParser(usage=usage)
parser.add_option('--etc', dest='etc',
default=DEFAULT_ETC,
help='Overrides /etc/swiftlm (for testing)')
parser.add_option('--input-vars', dest='input_vars', default=None,
help='The name of yaml containing vars')
parser.add_option('--builder-dir', dest='builder_dir', default=None,
help='Builder file directory')
parser.add_option('--osconfig', dest='osconfig', default=None,
help='Directory where drive size data files exist.'
' If omitted, the correct weights cannot'
' be assigned to drives.')
parser.add_option('--hosts', dest='hosts', default=None,
help='Hosts file (/etc/hosts or ./net/hosts.hf)')
parser.add_option('--swift-ring-builder-consumes',
dest='swift_ring_builder_consumes',
default=None,
help='File containing the SWF_RNG variable')
parser.add_option('--ring-delta', dest='ring_delta', default=None,
help='Name of ring-delta file (as output or input'
' A value of "-" (on output means to write'
' to stdout')
parser.add_option('--format', dest='fmt', default='yaml',
help='One of yaml or json.'
' When used with --ring-delta, specifies the'
' format of the file.')
parser.add_option('--detail', dest='detail', default='summary',
help='Level of detail to use with --report.'
' Use summary or full')
parser.add_option('--report', dest='report', default=False,
action="store_true",
help='Explain what the ring delta represents.'
' Optionally use --detail.')
parser.add_option('--dry-run', dest='dry_run', default=False,
action="store_true",
help='Show the proposed swift-ring-builder commands')
parser.add_option('--make-delta', dest='make_delta', default=False,
action="store_true",
help='Make a ring delta file')
parser.add_option('--rebalance', dest='rebalance', default=False,
action="store_true",
help='Build (or rebalance) rings')
parser.add_option('--storage-policies', dest='storage_policies',
default=False,
action="store_true",
help='Extract storage policy data.'
' Use this to register the storage_policies'
' variable.')
parser.add_option('--region-name', dest='region_name', default=None,
help='Region name for use with --storage-policies')
parser.add_option('--size-to-weight', dest='size_to_weight',
default=float(1024 * 1024 * 1024),
help='Conversion factor for size to weight. Default is'
' 1GB is weight of 1 (a 4Tb drive would be assigned'
' a weight of 4096')
parser.add_option('--weight-step', dest='weight_step',
default=None,
help='When set, weights are changed by at most this'
' value. Overrides value in ring specification.')
parser.add_option('--allow-partitions', dest='allow_partitions',
default=False, action='store_true',
help='Allow devices to be assigned to partitions.'
' Default is to use a full disk drive.')
parser.add_option('--stop-on-warnings', dest='stop_on_warnings',
default=False, action='store_true',
help='Used with --make-delta. Exit with error if there'
' are model missmatch warnings.'
' Default is to only exit with error for errors.')
(options, args) = parser.parse_args()
if not options.input_vars:
options.input_vars = os.path.join(options.etc, DEFAULT_INPUT_VARS)
if not options.builder_dir:
options.builder_dir = os.path.join(options.etc, DEFAULT_BUILDER_DIR)
if not options.hosts:
options.hosts = os.path.join(options.etc, DEFAULT_HOSTS)
if not options.swift_ring_builder_consumes:
options.swift_ring_builder_consumes = os.path.join(
options.etc, DEFAULT_SWIFT_RING_BUILDER_CONSUMES)
if not options.ring_delta:
options.ring_delta = os.path.join(options.etc, DEFAULT_RING_DELTA)
if not options.osconfig:
options.osconfig = os.path.join(options.etc, DEFAULT_OSCONFIG)
#
# Work out what we need to do. Validate arguments needed by an action
# are present.
#
actions = []
if options.storage_policies:
actions.append('input-from-model')
actions.append('emit-storage-policies')
if not options.region_name:
print('Need --region-name')
sys.exit(1)
if options.make_delta:
actions.append('init-delta')
actions.append('input-from-model')
actions.append('read-builder-dir')
actions.append('open-osconfig-dir')
actions.append('make-delta')
actions.append('write-to-delta')
if not options.ring_delta:
print('Need --ring-delta file to write to')
sys.exit(1)
if not (options.input_vars and options.hosts and
options.osconfig and options.swift_ring_builder_consumes):
print('Need --input-vars, --hosts and --osconfig and'
' --swift-ring-builder-consumes inputs')
sys.exit(1)
if not options.builder_dir:
print('Need --builder-dir option')
sys.exit(1)
if options.fmt not in ['yaml', 'json']:
print('Invalid value for --format')
if options.report:
actions.append('init-delta')
actions.append('read-from-delta')
actions.append('report')
if not options.ring_delta:
print('Need --ring-delta file as input')
sys.exit(1)
if options.detail not in ['summary', 'full']:
print('Invalid value for --detail')
sys.exit(1)
if options.rebalance:
actions.append('init-delta')
actions.append('open-builder-dir')
actions.append('read-from-delta')
actions.append('rebalance')
if not options.ring_delta:
print('Need --ring-delta file as input')
sys.exit(1)
if options.fmt not in ['yaml', 'json']:
print('Invalid value for --format')
if len(actions) == 0:
print('Missing an option to perform some action')
sys.exit(1)
if options.storage_policies and (options.make_delta or
options.rebalance or
options.report):
print('Do not mix --storage-policies with other actions')
sys.exit(1)
if options.report and (options.make_delta or
options.rebalance or
options.storage_policies):
print('Do not mix --report with other actions')
sys.exit(1)
#
# Perform actions
#
if 'init-delta' in actions:
delta = RingDelta()
if 'input-from-model' in actions:
try:
input_model_fd = open(options.input_vars, 'r')
hosts_fd = open(options.hosts, 'r')
consumes_fd = open(options.swift_ring_builder_consumes, 'r')
except IOError as err:
print('ERROR: %s' % err)
sys.exit(1)
try:
input_vars = safe_load(input_model_fd)
consumes_model = safe_load(consumes_fd)
except scanner.ScannerError as err:
print('ERROR in %s: %s' % (options.input_vars, err))
sys.exit(err)
try:
input_model = InputModel(config=input_vars, hosts_fd=hosts_fd,
consumes=consumes_model)
ring_model = RingSpecifications(model=input_vars)
except SwiftModelException as err:
sys.exit(err)
if 'emit-storage-policies' in actions:
obj_ring_policies = ring_model.get_storage_policies(
options.region_name)
print('%s' % safe_dump(obj_ring_policies, default_flow_style=False))
# Unlike the following actions, we exit after this action
sys.exit(0)
if 'open-builder-dir' or 'read-builder-dir' in actions:
try:
read_write = False
if 'read-builder-dir' in actions:
read_write = True
rings = RingBuilder(options.builder_dir, True)
except IOError as err:
print('ERROR: %s' % err)
sys.exit(1)
if 'open-osconfig-dir' in actions:
drive_configurations = osconfig_load(options.osconfig)
if 'make-delta' in actions:
try:
generate_delta(input_model, ring_model, rings,
drive_configurations, options, delta)
except SwiftModelException as err:
print('ERROR: %s' % err)
sys.exit(1)
if 'write-to-delta' in actions:
if options.ring_delta == '-':
write_to_file_fd = sys.stdout
else:
write_to_file_fd = open(options.ring_delta, 'w')
delta.write_to_file(write_to_file_fd, options.fmt)
if 'read-from-delta' in actions:
if options.ring_delta == '-':
print('--ring-delta=- is invalid (read from stdin not supported)')
sys.exit(1)
try:
delta = RingDelta()
read_from_delta_fd = open(options.ring_delta, 'r')
delta.read_from_file(read_from_delta_fd, options.fmt)
except IOError as err:
print('ERROR: %s' % err)
sys.exit(1)
if 'report' in actions:
print(delta.get_report(options.detail))
if 'rebalance' in actions:
rebalance(delta, rings, options.dry_run)
def main():
parser = OptionParser(usage=usage)
parser.add_option('--etc',
dest='etc',
default=DEFAULT_ETC,
help='Overrides /etc/swiftlm (for testing)')
parser.add_option('--cloud',
dest='cloud',
default=None,
help='The name of the cloud')
parser.add_option('--control-plane',
dest='control_plane',
default=None,
help='The name of the control plane')
parser.add_option('--ring-delta',
dest='ring_delta',
default=None,
help='Name of ring-delta file (as output or input'
' A value of "-" (on output means to write'
' to stdout')
parser.add_option('--format',
dest='fmt',
default='yaml',
help='One of yaml or json.'
' When used with --ring-delta, specifies the'
' format of the file.')
parser.add_option('--detail',
dest='detail',
default='summary',
help='Level of detail to use with --report.'
' Use summary or full')
parser.add_option('--report',
dest='report',
default=False,
action="store_true",
help='Explain what the ring delta represents.'
' Optionally use --detail.')
parser.add_option('--dry-run',
dest='dry_run',
default=False,
action="store_true",
help='Show the proposed swift-ring-builder commands')
parser.add_option('--pretend-min-part-hours-passed',
dest='pretend_min_part_hours_passed',
default=False,
action="store_true",
help='Executes the pretend_min_part_hours_passed command'
' on each ring before running rebalance.'
' Use with caution.')
parser.add_option('--make-delta',
dest='make_delta',
default=False,
action="store_true",
help='Make a ring delta file')
parser.add_option('--rebalance',
dest='rebalance',
default=False,
action="store_true",
help='Build (or rebalance) rings')
parser.add_option('--limit-ring',
dest='limit_ring',
default=None,
help='Limits actions to given ring')
parser.add_option('--size-to-weight',
dest='size_to_weight',
default=float(1024 * 1024 * 1024),
help='Conversion factor for size to weight. Default is'
' 1GB is weight of 1 (a 4Tb drive would be assigned'
' a weight of 4096')
parser.add_option('--weight-step',
dest='weight_step',
default=None,
help='When set, weights are changed by at most this'
' value. Overrides value in ring specification.')
parser.add_option('--allow-partitions',
dest='allow_partitions',
default=False,
action='store_true',
help='Allow devices to be assigned to partitions.'
' Default is to use a full disk drive.')
parser.add_option('--stop-on-warnings',
dest='stop_on_warnings',
default=False,
action='store_true',
help='Used with --make-delta. Exit with error if there'
' are model missmatch warnings.'
' Default is to only exit with error for errors.')
parser.add_option('--unittest',
dest='unittest',
default=False,
action='store_true',
help='Set by unittests. Never set on command line.')
(options, args) = parser.parse_args()
if not (options.cloud and options.control_plane):
sys.exit('Must specify both --cloud and --control_plane')
sites = CloudMultiSite(options)
my_cloud = sites.my_cloud
my_control_plane = sites.my_control_plane
my_config = sites.path(my_cloud, my_control_plane)
#
# Work out what we need to do. Validate arguments needed by an action
# are present.
#
actions = []
if options.make_delta:
actions.append('init-delta')
actions.append('input-from-model')
actions.append('read-builder-dir')
actions.append('open-osconfig-dir')
actions.append('make-delta')
actions.append('write-to-delta')
if options.fmt not in ['yaml', 'json']:
print('Invalid value for --format')
if options.report:
actions.append('init-delta')
actions.append('read-from-delta')
actions.append('report')
if options.detail not in ['summary', 'full']:
sys.exit('Invalid value for --detail')
if options.rebalance:
actions.append('init-delta')
actions.append('open-builder-dir')
actions.append('read-from-delta')
actions.append('rebalance')
if options.fmt not in ['yaml', 'json']:
print('Invalid value for --format')
if len(actions) == 0:
sys.exit('Missing an option to perform some action')
if options.report and (options.make_delta or options.rebalance):
sys.exit('Do not mix --report with other actions')
#
# Perform actions
#
if 'init-delta' in actions:
delta = RingDelta()
if 'input-from-model' in actions:
servers_model = ServersModel('unused', 'unused')
consumes = Consumes()
ring_model = RingSpecifications(my_cloud, my_control_plane)
for cloud, control_plane in sites.control_planes():
config = sites.path(cloud, control_plane)
input_model_fd = None
try:
input_model_fd = open(config.get('input-model'), 'r')
except IOError as err:
pass # File may not exist since its a legacy item
try:
cp_server_fd = None
cp_server_fd = open(config.get('control_plane_servers'), 'r')
except IOError as err:
sys.exit('Error on control_plane_server.yml: %s' % err)
try:
control_plane_servers = None
if cp_server_fd:
control_plane_servers = safe_load(cp_server_fd)
except scanner.ScannerError as err:
sys.exit('ERROR reading/parsing: %s' % err)
try:
consumes_fd = open(config.get('swift_ring_builder_consumes'),
'r')
except IOError as err:
sys.exit('ERROR: %s' % err)
try:
input_vars = {'global': {}}
if input_model_fd:
input_vars = safe_load(input_model_fd)
consumes_model = safe_load(consumes_fd)
except scanner.ScannerError as err:
sys.exit('ERROR reading/parsing: %s' % err)
try:
if control_plane_servers:
servers = control_plane_servers.get(
'control_plane_servers')
elif input_vars.get('global').get('all_servers'):
servers = input_vars.get('global').get('all_servers')
else:
sys.exit('No servers found in control plane')
servers_model.add_servers(cloud, control_plane, servers)
consumes.load_model(consumes_model)
except SwiftModelException as err:
sys.exit(err)
servers_model.register_consumes(consumes)
try:
config_data_fd = open(my_config.get('configuration_data'), 'r')
config_data = safe_load(config_data_fd)
except (IOError, scanner.ScannerError) as err:
sys.exit('Rings should be in configuration-data.'
' Using old configuration processor?')
try:
rings_loaded = False
if input_vars.get('global').get('all_ring_specifications'):
# Model contains Ardana old-style rings
ring_model = RingSpecifications(my_cloud,
my_control_plane,
model=input_vars)
rings_loaded = True
if config_data and config_data.get(
'control_plane_rings',
config_data.get('control-plane-rings')):
# Model contains new-style rings --- use instead
ring_model.load_configuration(my_cloud, my_control_plane,
config_data)
rings_loaded = True
if not rings_loaded:
sys.exit('No ring specifications in input model')
except SwiftModelException as err:
sys.exit(err)
if 'open-builder-dir' or 'read-builder-dir' in actions:
try:
read_rings = False
if 'read-builder-dir' in actions:
read_rings = True
rings = RingBuilder(my_config.get('builder_dir'),
read_rings=read_rings)
except IOError as err:
sys.exit('ERROR: %s' % err)
if 'open-osconfig-dir' in actions:
drive_configurations = osconfig_load(sites)
if 'make-delta' in actions:
try:
generate_delta(sites, servers_model, ring_model, rings,
drive_configurations, options, delta)
except SwiftModelException as err:
sys.exit('ERROR: %s' % err)
if 'write-to-delta' in actions:
if my_config.get('ring-delta') == '-':
write_to_file_fd = sys.stdout
else:
write_to_file_fd = open(my_config.get('ring-delta'), 'w')
delta.write_to_file(write_to_file_fd, options.fmt)
if 'read-from-delta' in actions:
if my_config.get('ring-delta') == '-':
sys.exit('--ring-delta- is invalid (read from stdin'
'not supported)')
try:
delta = RingDelta()
read_from_delta_fd = open(my_config.get('ring-delta'), 'r')
delta.read_from_file(read_from_delta_fd, options.fmt)
except IOError as err:
sys.exit('ERROR: %s' % err)
if 'report' in actions:
print(delta.get_report(options))
if 'rebalance' in actions:
rebalance(delta, rings, options)