def test_load(self):
rb = ring.RingBuilder(8, 3, 1)
devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1',
'meta': 'meta0'},
{'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1',
'meta': 'meta1'},
{'id': 2, 'region': 0, 'zone': 2, 'weight': 2,
'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1',
'meta': 'meta2'},
{'id': 3, 'region': 0, 'zone': 3, 'weight': 2,
'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1'}]
for d in devs:
rb.add_dev(d)
rb.rebalance()
real_pickle = pickle.load
try:
#test a legit builder
fake_pickle = Mock(return_value=rb)
fake_open = Mock(return_value=None)
pickle.load = fake_pickle
builder = RingBuilder.load('fake.builder', open=fake_open)
self.assertEquals(fake_pickle.call_count, 1)
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
self.assertEquals(builder, rb)
fake_pickle.reset_mock()
fake_open.reset_mock()
#test old style builder
fake_pickle.return_value = rb.to_dict()
pickle.load = fake_pickle
builder = RingBuilder.load('fake.builder', open=fake_open)
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
self.assertEquals(builder.devs, rb.devs)
fake_pickle.reset_mock()
fake_open.reset_mock()
#test old devs but no meta
no_meta_builder = rb
for dev in no_meta_builder.devs:
del(dev['meta'])
fake_pickle.return_value = no_meta_builder
pickle.load = fake_pickle
builder = RingBuilder.load('fake.builder', open=fake_open)
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
self.assertEquals(builder.devs, rb.devs)
fake_pickle.reset_mock()
finally:
pickle.load = real_pickle
def test_load(self):
rb = ring.RingBuilder(8, 3, 1)
devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1',
'meta': 'meta0'},
{'id': 1, 'region': 0, 'zone': 1, 'weight': 1,
'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1',
'meta': 'meta1'},
{'id': 2, 'region': 0, 'zone': 2, 'weight': 2,
'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1',
'meta': 'meta2'},
{'id': 3, 'region': 0, 'zone': 3, 'weight': 2,
'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1'}]
for d in devs:
rb.add_dev(d)
rb.rebalance()
real_pickle = pickle.load
try:
#test a legit builder
fake_pickle = Mock(return_value=rb)
fake_open = Mock(return_value=None)
pickle.load = fake_pickle
builder = RingBuilder.load('fake.builder', open=fake_open)
self.assertEquals(fake_pickle.call_count, 1)
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
self.assertEquals(builder, rb)
fake_pickle.reset_mock()
fake_open.reset_mock()
#test old style builder
fake_pickle.return_value = rb.to_dict()
pickle.load = fake_pickle
builder = RingBuilder.load('fake.builder', open=fake_open)
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
self.assertEquals(builder.devs, rb.devs)
fake_pickle.reset_mock()
fake_open.reset_mock()
#test old devs but no meta
no_meta_builder = rb
for dev in no_meta_builder.devs:
del(dev['meta'])
fake_pickle.return_value = no_meta_builder
pickle.load = fake_pickle
builder = RingBuilder.load('fake.builder', open=fake_open)
fake_open.assert_has_calls([mock_call('fake.builder', 'rb')])
self.assertEquals(builder.devs, rb.devs)
fake_pickle.reset_mock()
finally:
pickle.load = real_pickle
def rebalance(self, builder_type, lasthash, start_response, env):
""" rebalance a ring
note: rebalance doesn't yield.
"""
with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
self.verify_current_hash(self.bf_path[builder_type], lasthash)
builder = RingBuilder.load(self.bf_path[builder_type])
devs_changed = builder.devs_changed
try:
last_balance = builder.get_balance()
parts, balance = builder.rebalance()
except RingBuilderError, err:
self.logger.exception(_("Error during ring validation."))
return self.return_response(False, None, err.message,
start_response, env)
if not parts:
msg = 'Either none need to be assigned or none can be due ' \
'to min_part_hours [%s].' % builder.min_part_hours
self.logger.error(_(msg))
return self.return_response(False, None, msg, start_response,
env)
if not devs_changed and abs(last_balance - balance) < 1:
msg = 'Refusing to save rebalance. Did not change at least 1%.'
self.logger.error(_(msg))
return self.return_response(False, None, msg, start_response,
env)
try:
builder.validate()
except RingValidationError, err:
self.logger.exception(_("Error during ring validation."))
return self.return_response(False, None, err.message,
start_response, env)
def __init__(self, builder_file, swiftdir='/etc/swift',
backup_dir_name='backups', verbose=False):
self.verbose = verbose
self.builder_file = builder_file
self.swiftdir = swiftdir
self.backup_dir = pathjoin(self.swiftdir, backup_dir_name)
self.builder = RingBuilder.load(builder_file)
def test_remove_device(self):
for search_value in self.search_values:
self.create_sample_ring()
argv = ["", self.tmpfile, "remove", search_value]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
# Check that weight was set to 0
dev = [d for d in ring.devs if d['id'] == 0][0]
self.assertEqual(dev['weight'], 0)
# Check that device is in list of devices to be removed
dev = [d for d in ring._remove_devs if d['id'] == 0][0]
self.assertEqual(dev['region'], 0)
self.assertEqual(dev['zone'], 0)
self.assertEqual(dev['ip'], '127.0.0.1')
self.assertEqual(dev['port'], 6000)
self.assertEqual(dev['device'], 'sda1')
self.assertEqual(dev['weight'], 0)
self.assertEqual(dev['replication_ip'], '127.0.0.1')
self.assertEqual(dev['replication_port'], 6000)
self.assertEqual(dev['meta'], 'some meta data')
# Check that second device in ring is not affected
dev = [d for d in ring.devs if d['id'] == 1][0]
self.assertEqual(dev['weight'], 100)
self.assertFalse([d for d in ring._remove_devs if d['id'] == 1])
# Final check, rebalance and check ring is ok
ring.rebalance()
self.assertTrue(ring.validate())
def test_validate(self):
self.create_sample_ring()
ring = RingBuilder.load(self.tmpfile)
ring.rebalance()
ring.save(self.tmpfile)
argv = ["", self.tmpfile, "validate"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
def test_create_ring(self):
argv = ["", self.tmpfile, "create", "6", "3.14159265359", "1"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.part_power, 6)
self.assertEqual(ring.replicas, 3.14159265359)
self.assertEqual(ring.min_part_hours, 1)
def remove_devs(self, builder_type, devices, lasthash, start_response,
env):
""" remove devices from the builder
:params builder_type: the builder_type to use when loading the builder
:params devices: list of device ids to be removed.
:params lasthash: the hash to use when verifying state
"""
with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
self.verify_current_hash(self.bf_path[builder_type], lasthash)
builder = RingBuilder.load(self.bf_path[builder_type])
if not isinstance(devices, list):
return self.return_response(False, lasthash,
'Malformed request.',
start_response, env)
for dev_id in devices:
sleep() # so we don't starve/block
try:
builder.remove_dev(int(dev_id))
except (IndexError, TypeError):
return self.return_response(False, lasthash,
'Invalid dev id %s.' % dev_id,
start_response, env)
except RingBuilderError as err:
return self.return_response(False, lasthash,
'Error removing %s - %s.' %
(dev_id, err),
start_response, env)
except ValueError as err:
return self.return_response(False, lasthash, str(err),
start_response, env)
newmd5 = self.write_builder(builder, self.bf_path[builder_type])
return self.return_response(True, newmd5, None, start_response,
env)
def test_create_ring(self):
argv = ["", self.tmpfile, "create", "6", "3.14159265359", "1"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.part_power, 6)
self.assertEqual(ring.replicas, 3.14159265359)
self.assertEqual(ring.min_part_hours, 1)
def test_remove_device(self):
for search_value in self.search_values:
self.create_sample_ring()
argv = ["", self.tmpfile, "remove", search_value]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
# Check that weight was set to 0
dev = [d for d in ring.devs if d['id'] == 0][0]
self.assertEqual(dev['weight'], 0)
# Check that device is in list of devices to be removed
dev = [d for d in ring._remove_devs if d['id'] == 0][0]
self.assertEqual(dev['region'], 0)
self.assertEqual(dev['zone'], 0)
self.assertEqual(dev['ip'], '127.0.0.1')
self.assertEqual(dev['port'], 6000)
self.assertEqual(dev['device'], 'sda1')
self.assertEqual(dev['weight'], 0)
self.assertEqual(dev['replication_ip'], '127.0.0.1')
self.assertEqual(dev['replication_port'], 6000)
self.assertEqual(dev['meta'], 'some meta data')
# Check that second device in ring is not affected
dev = [d for d in ring.devs if d['id'] == 1][0]
self.assertEqual(dev['weight'], 100)
self.assertFalse([d for d in ring._remove_devs if d['id'] == 1])
# Final check, rebalance and check ring is ok
ring.rebalance()
self.assertTrue(ring.validate())
def test_set_info(self):
for search_value in self.search_values:
self.create_sample_ring()
argv = ["", self.tmpfile, "set_info", search_value,
"127.0.1.1:8000/sda1_other meta data"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
# Check that device was created with given data
ring = RingBuilder.load(self.tmpfile)
dev = [d for d in ring.devs if d['id'] == 0][0]
self.assertEqual(dev['ip'], '127.0.1.1')
self.assertEqual(dev['port'], 8000)
self.assertEqual(dev['device'], 'sda1')
self.assertEqual(dev['meta'], 'other meta data')
# Check that second device in ring is not affected
dev = [d for d in ring.devs if d['id'] == 1][0]
self.assertEqual(dev['ip'], '127.0.0.2')
self.assertEqual(dev['port'], 6001)
self.assertEqual(dev['device'], 'sda2')
self.assertEqual(dev['meta'], '')
# Final check, rebalance and check ring is ok
ring.rebalance()
self.assertTrue(ring.validate())
def test_set_info(self):
for search_value in self.search_values:
self.create_sample_ring()
argv = ["", self.tmpfile, "set_info", search_value,
"127.0.1.1:8000/sda1_other meta data"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
# Check that device was created with given data
ring = RingBuilder.load(self.tmpfile)
dev = [d for d in ring.devs if d['id'] == 0][0]
self.assertEqual(dev['ip'], '127.0.1.1')
self.assertEqual(dev['port'], 8000)
self.assertEqual(dev['device'], 'sda1')
self.assertEqual(dev['meta'], 'other meta data')
# Check that second device in ring is not affected
dev = [d for d in ring.devs if d['id'] == 1][0]
self.assertEqual(dev['ip'], '127.0.0.2')
self.assertEqual(dev['port'], 6001)
self.assertEqual(dev['device'], 'sda2')
self.assertEqual(dev['meta'], '')
# Final check, rebalance and check ring is ok
ring.rebalance()
self.assertTrue(ring.validate())
def add_to_ring(self, builder_type, body, lasthash, start_response, env):
""" Handle a add device post """
with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
self.verify_current_hash(self.bf_path[builder_type], lasthash)
builder = RingBuilder.load(self.bf_path[builder_type])
ring_modified = False
try:
for device in body['devices']:
sleep() # so we don't starve/block
if not self._is_existing_dev(builder, device['ip'],
int(device['port']),
device['device']):
self._add_device(builder, int(device['zone']),
device['ip'], int(device['port']),
device['device'],
float(device['weight']),
device['meta'])
ring_modified = True
except (AttributeError, KeyError, ValueError, TypeError) as err:
return self.return_response(False, lasthash,
"Malformed request.",
start_response, env)
if ring_modified:
newmd5 = self.write_builder(builder,
self.bf_path[builder_type])
return self.return_response(True, newmd5, None,
start_response, env)
else:
return self.return_response(False, lasthash,
'Ring remains unchanged.',
start_response, env)
def change_meta(self, builder_type, dev_meta, lasthash, start_response,
env):
""" Change meta info for devices
:param builder_type: the builder_type to use when loading the builder
:param dev_meta: a dict of device id and meta info
:param lasthash: the hash to use when verifying state
"""
with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
self.verify_current_hash(self.bf_path[builder_type], lasthash)
builder = RingBuilder.load(self.bf_path[builder_type])
try:
modified = False
for dev_id in dev_meta:
sleep() # so we don't starve/block
for device in builder.devs:
if not device:
continue
if device['id'] == int(dev_id):
modified = True
device['meta'] = '%s' % dev_meta[dev_id]
if modified:
newmd5 = self.write_builder(builder,
self.bf_path[builder_type])
return self.return_response(True, newmd5, None,
start_response, env)
else:
return self.return_response(False, lasthash,
'Invalid dev id %s.' % dev_id,
start_response, env)
except ValueError as err:
return self.return_response(False, lasthash, str(err),
start_response, env)
def change_weight(self, builder_type, dev_weights, lasthash,
start_response, env):
""" Change weight of devices
:param builder_type: the builder_type to use when loading the builder
:param dev_weights: a dict of device id and weight
:param lasthash: the hash to use when verifying state
"""
with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
self.verify_current_hash(self.bf_path[builder_type], lasthash)
builder = RingBuilder.load(self.bf_path[builder_type])
for dev_id in dev_weights:
sleep() # so we don't starve/block
try:
builder.set_dev_weight(int(dev_id),
float(dev_weights[dev_id]))
except (IndexError, TypeError):
return self.return_response(False, lasthash,
'Invalid dev id %s.' % dev_id,
start_response, env)
except ValueError as err:
return self.return_response(False, lasthash, str(err),
start_response, env)
newmd5 = self.write_builder(builder, self.bf_path[builder_type])
return self.return_response(True, newmd5, None, start_response,
env)
def test_validate(self):
self.create_sample_ring()
ring = RingBuilder.load(self.tmpfile)
ring.rebalance()
ring.save(self.tmpfile)
argv = ["", self.tmpfile, "validate"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
def test_server_overload_crazy_high(self):
self.create_sample_ring()
argv = "set_overload 10".split()
out, err = self.run_srb(*argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.overload, 10.0)
self.assertTrue('Warning overload is greater than 100%' in out)
self.assertTrue('1000.00%' in out)
self.assertTrue('10.000000' in out)
# but it's cool if you do it on purpose
argv[-1] = '1000%'
out, err = self.run_srb(*argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.overload, 10.0)
self.assertTrue('Warning overload is greater than 100%' not in out)
self.assertTrue('1000.00%' in out)
self.assertTrue('10.000000' in out)
def test_load(self):
rb = ring.RingBuilder(8, 3, 1)
devs = [
{"id": 0, "zone": 0, "weight": 1, "ip": "127.0.0.0", "port": 10000, "device": "sda1", "meta": "meta0"},
{"id": 1, "zone": 1, "weight": 1, "ip": "127.0.0.1", "port": 10001, "device": "sdb1", "meta": "meta1"},
{"id": 2, "zone": 2, "weight": 2, "ip": "127.0.0.2", "port": 10002, "device": "sdc1", "meta": "meta2"},
{"id": 3, "zone": 3, "weight": 2, "ip": "127.0.0.3", "port": 10003, "device": "sdd1"},
]
for d in devs:
rb.add_dev(d)
rb.rebalance()
real_pickle = pickle.load
try:
# test a legit builder
fake_pickle = Mock(return_value=rb)
fake_open = Mock(return_value=None)
pickle.load = fake_pickle
builder = RingBuilder.load("fake.builder", open=fake_open)
self.assertEquals(fake_pickle.call_count, 1)
fake_open.assert_has_calls([mock_call("fake.builder", "rb")])
self.assertEquals(builder, rb)
fake_pickle.reset_mock()
fake_open.reset_mock()
# test old style builder
fake_pickle.return_value = rb.to_dict()
pickle.load = fake_pickle
builder = RingBuilder.load("fake.builder", open=fake_open)
fake_open.assert_has_calls([mock_call("fake.builder", "rb")])
self.assertEquals(builder.devs, rb.devs)
fake_pickle.reset_mock()
fake_open.reset_mock()
# test old devs but no meta
no_meta_builder = rb
for dev in no_meta_builder.devs:
del (dev["meta"])
fake_pickle.return_value = no_meta_builder
pickle.load = fake_pickle
builder = RingBuilder.load("fake.builder", open=fake_open)
fake_open.assert_has_calls([mock_call("fake.builder", "rb")])
self.assertEquals(builder.devs, rb.devs)
fake_pickle.reset_mock()
finally:
pickle.load = real_pickle
def test_set_overload_percent_strange_input(self):
self.create_sample_ring()
argv = "set_overload 26%%%%".split()
out, err = self.run_srb(*argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.overload, 0.26)
self.assertTrue('26.00%' in out)
self.assertTrue('0.260000' in out)
def test_set_overload_percent(self):
self.create_sample_ring()
argv = "set_overload 10%".split()
out, err = self.run_srb(*argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.overload, 0.1)
self.assertTrue('10.00%' in out)
self.assertTrue('0.100000' in out)
def _get_devs_from_builder(self, builder_filename):
ring_name = builder_filename[0:builder_filename.find('.builder')]
try:
builder = SwiftRingBuilder.load(builder_filename)
except Exception as e:
raise IOError('ERROR: swift-ring-builder Problem occurred while '
'reading builder file: %s. %s' %
(builder_filename, e))
self.partitions = builder.parts
self.replica_count = builder.replicas
balance = 0
if builder.devs:
balance = builder.get_balance()
self.balance = balance
self.dispersion = 0.00
if builder.dispersion:
self.dispersion = float(builder.dispersion)
self.min_part_hours = builder.min_part_hours
self.remaining = str(timedelta(seconds=builder.min_part_seconds_left))
self.overload = builder.overload
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
for dev in builder._iter_devs():
if dev in builder._remove_devs:
# This device was added and then later removed without
# doing a rebalance in the meantime. We can ignore
# since it's marked for deletion.
continue
device_info = DeviceInfo({
'ring_name':
ring_name,
'zone_id':
dev['zone'],
'region_id':
dev['region'],
'server_ip':
dev['ip'],
'server_bind_port':
dev['port'],
'replication_ip':
dev['replication_ip'],
'replication_bind_port':
dev['replication_port'],
'swift_drive_name':
dev['device'],
'current_weight':
dev['weight'],
'balance':
balance_per_dev[dev['id']],
'meta':
dev['meta'],
'presence':
'present'
})
yield device_info
def check_builder(builder_file, save_builder=False, fix_replicas=False,
set_overload=None, set_min_part_hours=None, **kwargs):
"""
Create a builder from builder_file and rebalance, return the stats.
:param builder_file: path to builder on disk
:param save_builder: bool, if true save builder after rebalance
:param fix_replicas: bool, if true reduce replica count on ring to
max(count_of_devices_with_weight,
current_replica_count)
:param set_overload: float or None, if float set_overload on the builder
before rebalance
:param set_min_part_hours: int or None, if int set_min_part_hours on the
builder before rebalance
:returns: stats, a dict, information about the check
"""
builder = RingBuilder.load(builder_file)
count_of_devices_with_weight = len([d for d in builder._iter_devs()
if d['weight'] > 0])
stats = {
'builder_file': builder_file,
'parts': builder.parts,
'replicas': builder.replicas,
'num_devs': count_of_devices_with_weight,
'overload': builder.overload,
'min_part_hours': builder.min_part_hours,
}
if count_of_devices_with_weight < 1:
return stats
builder._build_dispersion_graph()
stats.update({
'initial_balance': builder.get_balance(),
'initial_dispersion': builder.dispersion,
})
if fix_replicas:
if builder.replicas > count_of_devices_with_weight:
builder.set_replicas(float(count_of_devices_with_weight))
if set_overload is not None:
builder.set_overload(set_overload)
if set_min_part_hours is not None:
builder.change_min_part_hours(set_min_part_hours)
start = time.time()
parts_moved, final_balance = builder.rebalance()[:2]
builder.validate()
if save_builder:
builder.save(builder_file)
stats.update({
'final_replicas': builder.replicas,
'final_overload': builder.overload,
'final_min_part_hours': builder.min_part_hours,
'parts_moved': parts_moved,
'final_balance': final_balance,
'final_dispersion': builder.dispersion,
'rebalance_time': time.time() - start,
})
return stats
def main(arguments=None):
global argv, backup_dir, builder, builder_file, ring_file
if arguments:
argv = arguments
else:
argv = sys_argv
if len(argv) < 2:
print "swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" % \
globals()
print Commands.default.__doc__.strip()
print
cmds = [c for c, f in Commands.__dict__.iteritems()
if f.__doc__ and c[0] != '_' and c != 'default']
cmds.sort()
for cmd in cmds:
print Commands.__dict__[cmd].__doc__.strip()
print
print parse_search_value.__doc__.strip()
print
for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ',
subsequent_indent=' '):
print line
print ('Exit codes: 0 = operation successful\n'
' 1 = operation completed with warnings\n'
' 2 = error')
exit(EXIT_SUCCESS)
builder_file, ring_file = parse_builder_ring_filename_args(argv)
if exists(builder_file):
builder = RingBuilder.load(builder_file)
elif len(argv) < 3 or argv[2] not in('create', 'write_builder'):
print 'Ring Builder file does not exist: %s' % argv[1]
exit(EXIT_ERROR)
backup_dir = pathjoin(dirname(argv[1]), 'backups')
try:
mkdir(backup_dir)
except OSError as err:
if err.errno != EEXIST:
raise
if len(argv) == 2:
command = "default"
else:
command = argv[2]
if argv[0].endswith('-safe'):
try:
with lock_parent_directory(abspath(argv[1]), 15):
Commands.__dict__.get(command, Commands.unknown.im_func)()
except exceptions.LockTimeout:
print "Ring/builder dir currently locked."
exit(2)
else:
Commands.__dict__.get(command, Commands.unknown.im_func)()
def main(arguments=None):
global argv, backup_dir, builder, builder_file, ring_file
if arguments:
argv = arguments
else:
argv = sys_argv
if len(argv) < 2:
print "swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" % \
globals()
print Commands.default.__doc__.strip()
print
cmds = [c for c, f in Commands.__dict__.iteritems()
if f.__doc__ and c[0] != '_' and c != 'default']
cmds.sort()
for cmd in cmds:
print Commands.__dict__[cmd].__doc__.strip()
print
print parse_search_value.__doc__.strip()
print
for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ',
subsequent_indent=' '):
print line
print('Exit codes: 0 = operation successful\n'
' 1 = operation completed with warnings\n'
' 2 = error')
exit(EXIT_SUCCESS)
builder_file, ring_file = parse_builder_ring_filename_args(argv)
if exists(builder_file):
builder = RingBuilder.load(builder_file)
elif len(argv) < 3 or argv[2] not in('create', 'write_builder'):
print 'Ring Builder file does not exist: %s' % argv[1]
exit(EXIT_ERROR)
backup_dir = pathjoin(dirname(argv[1]), 'backups')
try:
mkdir(backup_dir)
except OSError as err:
if err.errno != EEXIST:
raise
if len(argv) == 2:
command = "default"
else:
command = argv[2]
if argv[0].endswith('-safe'):
try:
with lock_parent_directory(abspath(argv[1]), 15):
Commands.__dict__.get(command, Commands.unknown.im_func)()
except exceptions.LockTimeout:
print "Ring/builder dir currently locked."
exit(2)
else:
Commands.__dict__.get(command, Commands.unknown.im_func)()
def test_warn_at_risk(self):
self.create_sample_ring()
ring = RingBuilder.load(self.tmpfile)
ring.devs[0]['weight'] = 10
ring.save(self.tmpfile)
argv = ["", self.tmpfile, "rebalance"]
try:
swift.cli.ringbuilder.main(argv)
except SystemExit as e:
self.assertEquals(e.code, 1)
def __init__(self,
builder_file,
swiftdir='/etc/swift',
backup_dir_name='backups',
verbose=False):
self.verbose = verbose
self.builder_file = builder_file
self.swiftdir = swiftdir
self.backup_dir = pathjoin(self.swiftdir, backup_dir_name)
self.builder = RingBuilder.load(builder_file)
def deploy_storage_policy(request, storage_policy_id):
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
key = "storage-policy:" + storage_policy_id
if request.method == "POST":
if r.exists(key):
try:
tmp_policy_file = get_policy_file_path(
settings.SWIFT_CFG_TMP_DIR, storage_policy_id)
deploy_policy_file = get_policy_file_path(
settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id)
deploy_gzip_filename = deploy_policy_file.replace(
'builder', 'ring.gz')
ring = RingBuilder.load(tmp_policy_file)
ring.rebalance()
ring.save(tmp_policy_file)
ringdata = ring.get_ring()
ringdata.save(deploy_gzip_filename)
data = r.hgetall(key)
update_sp_files(
settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id, {
'name': data['name'],
'deprecated': data['deprecated'],
'default': data['default']
})
copyfile(tmp_policy_file, deploy_policy_file)
rsync_dir_with_nodes(settings.SWIFT_CFG_DEPLOY_DIR,
'/etc/swift')
r.hset(key, 'deployed', 'True')
return JSONResponse('Storage policy deployed correctly',
status=status.HTTP_200_OK)
except RedisError:
return JSONResponse('Storage policy could not be deployed',
status=status.HTTP_400_BAD_REQUEST)
except exceptions.RingBuilderError, e:
return JSONResponse(
'Storage policy could not be deployed. Error message: %s' %
e.message,
status=status.HTTP_400_BAD_REQUEST)
else:
return JSONResponse('Storage policy not found.',
status=status.HTTP_404_NOT_FOUND)
def deploy_storage_policy(request, storage_policy_id):
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
key = "storage-policy:" + storage_policy_id
if request.method == "POST":
if r.exists(key):
try:
tmp_policy_file = get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, storage_policy_id)
deploy_policy_file = get_policy_file_path(settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id)
deploy_gzip_filename = deploy_policy_file.replace('builder', 'ring.gz')
ring = RingBuilder.load(tmp_policy_file)
ring.rebalance()
ring.save(tmp_policy_file)
ringdata = ring.get_ring()
ringdata.save(deploy_gzip_filename)
data = r.hgetall(key)
policy = {'name': data['name'],
'deprecated': data['deprecated'],
'default': data['default']}
if data['policy_type'] == 'EC':
policy.update({'policy_type': 'erasure_coding',
'ec_type': data['ec_type'],
'ec_num_data_fragments': data['ec_num_data_fragments'],
'ec_num_parity_fragments': data['ec_num_parity_fragments'],
'ec_object_segment_size': data['ec_object_segment_size'],
'ec_duplication_factor': data['ec_duplication_factor']})
update_sp_files(settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id, policy)
else:
update_sp_files(settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id, policy)
copyfile(tmp_policy_file, deploy_policy_file)
rsync_dir_with_nodes(settings.SWIFT_CFG_DEPLOY_DIR, '/etc/swift')
r.hset(key, 'deployed', 'True')
return JSONResponse('Storage policy deployed correctly', status=status.HTTP_200_OK)
except RedisError:
return JSONResponse('Storage policy could not be deployed', status=status.HTTP_400_BAD_REQUEST)
except exceptions.RingBuilderError as e:
return JSONResponse('Storage policy could not be deployed. Error message: %s' % e.message, status=status.HTTP_400_BAD_REQUEST)
else:
return JSONResponse('Storage policy not found.', status=status.HTTP_404_NOT_FOUND)
return JSONResponse('Only HTTP POST requests allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def list_devices(self, builder_type, start_response, env):
""" list ALL devices in the ring
:params builder_type: the builder_type to use when loading the builder
:returns: list of boolean status, md5sum of the current ring, and all
builder.devs
"""
with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
builder = RingBuilder.load(self.bf_path[builder_type])
current_md5sum = self._get_md5sum(self.bf_path[builder_type])
return self.return_response(True, current_md5sum, builder.devs,
start_response, env)
def delete_storage_policy_disks(request, storage_policy_id, disk_id):
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
key = "storage-policy:" + storage_policy_id
if request.method == 'DELETE':
if r.exists(key):
try:
tmp_policy_file = get_policy_file_path(
settings.SWIFT_CFG_TMP_DIR, storage_policy_id)
found = False
storage_policy = r.hgetall(key)
storage_policy['devices'] = json.loads(
storage_policy['devices'])
for i, disk in enumerate(storage_policy['devices']):
if disk_id == disk[0]:
found = True
ring = RingBuilder.load(tmp_policy_file)
ring.remove_dev(disk[1])
ring.save(tmp_policy_file)
del storage_policy['devices'][i]
storage_policy['devices'] = json.dumps(
storage_policy['devices'])
r.hset(key, 'devices', storage_policy['devices'])
r.hset(key, 'deployed', False)
return JSONResponse("Disk removed",
status=status.HTTP_204_NO_CONTENT)
if not found:
return JSONResponse('Disk not found',
status=status.HTTP_404_NOT_FOUND)
except RedisError:
return JSONResponse("Error updating storage policy",
status=status.HTTP_400_BAD_REQUEST)
else:
return JSONResponse('Storage policy not found.',
status=status.HTTP_404_NOT_FOUND)
return JSONResponse('Method not allowed.',
status=status.HTTP_405_METHOD_NOT_ALLOWED)
def test_validate_corrupted_file(self):
self.create_sample_ring()
ring = RingBuilder.load(self.tmpfile)
ring.rebalance()
self.assertTrue(ring.validate()) # ring is valid until now
ring.save(self.tmpfile)
argv = ["", self.tmpfile, "validate"]
# corrupt the file
with open(self.tmpfile, 'wb') as f:
f.write(os.urandom(1024))
try:
swift.cli.ringbuilder.main(argv)
except SystemExit as e:
self.assertEquals(e.code, 2)
def test_validate_corrupted_file(self):
self.create_sample_ring()
ring = RingBuilder.load(self.tmpfile)
ring.rebalance()
self.assertTrue(ring.validate()) # ring is valid until now
ring.save(self.tmpfile)
argv = ["", self.tmpfile, "validate"]
# corrupt the file
with open(self.tmpfile, 'wb') as f:
f.write(os.urandom(1024))
try:
swift.cli.ringbuilder.main(argv)
except SystemExit as e:
self.assertEquals(e.code, 2)
def compose(self, builder_files=None, force=False):
"""
Builds a composite ring using component ring builders loaded from a
list of builder files.
If a list of component ring builder files is given then that will be
used to load component ring builders. Otherwise, component ring
builders will be loaded using the list of builder files that was set
when the instance was constructed.
In either case, if metadata for an existing composite ring has been
loaded then the component ring builders are verified for consistency
with the existing composition of builders, unless the optional
``force`` flag if set True.
:param builder_files: Optional list of paths to ring builder
files that will be used to load the component ring builders.
Typically the list of component builder files will have been set
when the instance was constructed, for example when using the
load() class method. However, this parameter may be used if the
component builder file paths have moved, or, in conjunction with
the ``force`` parameter, if a new list of component builders is to
be used.
:param force: if True then do not verify given builders are consistent
with any existing composite ring.
:return: An instance of :class:`swift.common.ring.ring.RingData`
:raises: ValueError if the component ring builders are not suitable for
composing with each other, or are inconsistent with any existing
composite ring, or if there has been no change with respect to the
existing ring.
"""
builder_files = builder_files or self._builder_files
builders = [RingBuilder.load(f) for f in builder_files]
check_builder_ids(builders)
new_metadata = _make_composite_metadata(builders)
if self.components and self._builder_files and not force:
modified = check_against_existing(self.to_dict(), new_metadata)
if not modified:
raise ValueError(
"None of the component builders has been modified"
" since the existing composite ring was built.")
self.ring_data = compose_rings(builders)
self.version += 1
self.components = new_metadata['components']
self._set_builder_files(builder_files)
return self.ring_data
def write_ring(args, devices, builderfile):
# Make an educated guess about the used port. These are the defaults for
# TripleO-based deployments in Mitaka
builder_fname = os.path.basename(builderfile)
if 'account' in builder_fname:
port = 6002
elif 'container' in builder_fname:
port = 6001
elif 'object' in builder_fname:
port = 6000
else:
port = 6000
logging.debug('Set port for new devices to %d' % port)
if not os.path.isfile(builderfile):
logging.info(
'%s not found, creating new builder file', builderfile)
rb = RingBuilder(args.part_power, args.replicas, args.min_part_hours)
else:
logging.info('Using existing builder file %s', builderfile)
rb = RingBuilder.load(builderfile)
# Add all missing devices
for dev in devices:
_dev = rb.search_devs({'meta': dev['meta']})
if not _dev:
dev['weight'] = float(dev.get('size')) / 10**9
dev['region'] = 1
dev['zone'] = 1
dev['port'] = port
dev['replication_ip'] = dev['ip']
dev['replication_port'] = dev['port']
rb.add_dev(dev)
logging.info('Added device %s / %s', dev['ip'], dev['device'])
else:
logging.info(
'Ignoring existing device %s / %s', dev['ip'], dev['device'])
try:
rb.rebalance()
except RingValidationError as exc:
logging.error(exc)
rb.save(builderfile)
ring_file = os.path.splitext(builderfile)[0] + '.ring.gz'
ring_data = rb.get_ring()
ring_data.save(ring_file)
return [builderfile, ring_file]
def write_ring(args, devices, builderfile):
# Make an educated guess about the used port. These are the defaults for
# TripleO-based deployments in Mitaka
builder_fname = os.path.basename(builderfile)
if 'account' in builder_fname:
port = 6002
elif 'container' in builder_fname:
port = 6001
elif 'object' in builder_fname:
port = 6000
else:
port = 6000
logging.debug('Set port for new devices to %d' % port)
if not os.path.isfile(builderfile):
logging.info('%s not found, creating new builder file', builderfile)
rb = RingBuilder(args.part_power, args.replicas, args.min_part_hours)
else:
logging.info('Using existing builder file %s', builderfile)
rb = RingBuilder.load(builderfile)
# Add all missing devices
for dev in devices:
_dev = rb.search_devs({'meta': dev['meta']})
if not _dev:
dev['weight'] = float(dev.get('size')) / 10**9
dev['region'] = 1
dev['zone'] = 1
dev['port'] = port
dev['replication_ip'] = dev['ip']
dev['replication_port'] = dev['port']
rb.add_dev(dev)
logging.info('Added device %s / %s', dev['ip'], dev['device'])
else:
logging.info('Ignoring existing device %s / %s', dev['ip'],
dev['device'])
try:
rb.rebalance()
except RingValidationError as exc:
logging.error(exc)
rb.save(builderfile)
ring_file = os.path.splitext(builderfile)[0] + '.ring.gz'
ring_data = rb.get_ring()
ring_data.save(ring_file)
return [builderfile, ring_file]
def check_builder(builder_file, save_builder=False, fix_replicas=False,
**kwargs):
"""
Create a builder from builder_file and rebalance, return the stats.
:param builder_file: path to builder on disk
:param save_builder: bool, if true save builder after rebalance
:param fix_replicas: bool, if true reduce replica count on ring to
max(count_of_devices_with_weight,
current_replica_count)
:returns: stats, a dict, information about the check
"""
builder = RingBuilder.load(builder_file)
count_of_devices_with_weight = len([d for d in builder._iter_devs()
if d['weight'] > 0])
stats = {
'builder_file': builder_file,
'parts': builder.parts,
'replicas': builder.replicas,
'num_devs': count_of_devices_with_weight,
}
if count_of_devices_with_weight < 1:
return stats
builder._build_dispersion_graph()
stats.update({
'initial_balance': builder.get_balance(),
'initial_dispersion': builder.dispersion,
})
if fix_replicas:
if builder.replicas > count_of_devices_with_weight:
builder.set_replicas(float(count_of_devices_with_weight))
start = time.time()
parts_moved, final_balance = builder.rebalance()[:2]
builder.validate()
if save_builder:
builder.save(builder_file)
stats.update({
'final_replicas': builder.replicas,
'parts_moved': parts_moved,
'final_balance': final_balance,
'final_dispersion': builder.dispersion,
'rebalance_time': time.time() - start,
})
return stats
def test_set_weight(self):
for search_value in self.search_values:
self.create_sample_ring()
argv = ["", self.tmpfile, "set_weight", search_value, "3.14159265359"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
# Check that weight was changed
dev = [d for d in ring.devs if d["id"] == 0][0]
self.assertEqual(dev["weight"], 3.14159265359)
# Check that second device in ring is not affected
dev = [d for d in ring.devs if d["id"] == 1][0]
self.assertEqual(dev["weight"], 100)
# Final check, rebalance and check ring is ok
ring.rebalance()
self.assertTrue(ring.validate())
def test_set_weight(self):
for search_value in self.search_values:
self.create_sample_ring()
argv = ["", self.tmpfile, "set_weight",
search_value, "3.14159265359"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
# Check that weight was changed
dev = [d for d in ring.devs if d['id'] == 0][0]
self.assertEqual(dev['weight'], 3.14159265359)
# Check that second device in ring is not affected
dev = [d for d in ring.devs if d['id'] == 1][0]
self.assertEqual(dev['weight'], 100)
# Final check, rebalance and check ring is ok
ring.rebalance()
self.assertTrue(ring.validate())
def check_composite_meta(self, cb_file, builder_files, version=1):
with open(cb_file) as fd:
actual = json.load(fd)
builders = [RingBuilder.load(fname) for fname in builder_files]
expected_metadata = {
'saved_path': os.path.abspath(cb_file),
'serialization_version': 1,
'version': version,
'components': [
{'id': builder.id,
'version': builder.version,
'replicas': builder.replicas,
}
for builder in builders
],
'component_builder_files':
dict((builder.id, os.path.abspath(builder_files[i]))
for i, builder in enumerate(builders))
}
self.assertEqual(expected_metadata, actual)
def delete_storage_policy_disks(request, storage_policy_id, disk_id):
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
key = "storage-policy:" + storage_policy_id
if request.method == 'DELETE':
if r.exists(key):
try:
tmp_policy_file = get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, storage_policy_id)
found = False
storage_policy = r.hgetall(key)
storage_policy['devices'] = json.loads(storage_policy['devices'])
for i, disk in enumerate(storage_policy['devices']):
if disk_id == disk[0]:
found = True
ring = RingBuilder.load(tmp_policy_file)
ring.remove_dev(disk[1])
ring.save(tmp_policy_file)
del storage_policy['devices'][i]
storage_policy['devices'] = json.dumps(storage_policy['devices'])
r.hset(key, 'devices', storage_policy['devices'])
r.hset(key, 'deployed', False)
return JSONResponse("Disk removed", status=status.HTTP_204_NO_CONTENT)
if not found:
return JSONResponse('Disk not found', status=status.HTTP_404_NOT_FOUND)
except RedisError:
return JSONResponse("Error updating storage policy", status=status.HTTP_400_BAD_REQUEST)
else:
return JSONResponse('Storage policy not found.', status=status.HTTP_404_NOT_FOUND)
return JSONResponse('Method not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def test_add_device(self):
self.create_sample_ring()
argv = ["", self.tmpfile, "add", "r2z3-127.0.0.1:6000/sda3_some meta data", "3.14159265359"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
# Check that device was created with given data
ring = RingBuilder.load(self.tmpfile)
dev = [d for d in ring.devs if d["id"] == 2][0]
self.assertEqual(dev["region"], 2)
self.assertEqual(dev["zone"], 3)
self.assertEqual(dev["ip"], "127.0.0.1")
self.assertEqual(dev["port"], 6000)
self.assertEqual(dev["device"], "sda3")
self.assertEqual(dev["weight"], 3.14159265359)
self.assertEqual(dev["replication_ip"], "127.0.0.1")
self.assertEqual(dev["replication_port"], 6000)
self.assertEqual(dev["meta"], "some meta data")
# Final check, rebalance and check ring is ok
ring.rebalance()
self.assertTrue(ring.validate())
def search(self, builder_type, search_pattern, start_response, env):
""" search the builder for devices matching search pattern
:params builder_type: the builder_type to use when loading the builder
:params search_values: the value to search for
:returns: list of boolean status, md5sum of current builder
file on disk, and error message or dict of matched devices.
"""
with lock_file(self.bf_path[builder_type], timeout=1, unlink=False):
builder = RingBuilder.load(self.bf_path[builder_type])
try:
search_result = builder.search_devs(str(search_pattern))
return self.return_response(True, self._get_md5sum(
self.bf_path[builder_type]),
search_result,
start_response, env)
except ValueError:
return self.return_response(False, self._get_md5sum(
self.bf_path[builder_type]),
'Invalid search term',
start_response, env)
def test_cached_dispersion_value(self):
self.run_srb("create", 8, 3, 24)
self.run_srb("add",
"r1z1-10.1.1.1:2345/sda", 100.0,
"r1z1-10.1.1.1:2345/sdb", 100.0,
"r1z1-10.1.1.1:2345/sdc", 100.0,
"r1z1-10.1.1.1:2345/sdd", 100.0)
self.run_srb('rebalance')
out, err = self.run_srb() # list devices
self.assertTrue('dispersion' in out)
# remove cached dispersion value
builder = RingBuilder.load(self.tempfile)
builder.dispersion = None
builder.save(self.tempfile)
# now dispersion output is suppressed
out, err = self.run_srb() # list devices
self.assertFalse('dispersion' in out)
# but will show up after rebalance
self.run_srb('rebalance', '-f')
out, err = self.run_srb() # list devices
self.assertTrue('dispersion' in out)
def deploy_storage_policy(request, storage_policy_id):
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
key = "storage-policy:" + storage_policy_id
if request.method == "POST":
if r.exists(key):
try:
tmp_policy_file = get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, storage_policy_id)
deploy_policy_file = get_policy_file_path(settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id)
deploy_gzip_filename = deploy_policy_file.replace('builder', 'ring.gz')
ring = RingBuilder.load(tmp_policy_file)
ring.rebalance()
ring.save(tmp_policy_file)
ringdata = ring.get_ring()
ringdata.save(deploy_gzip_filename)
data = r.hgetall(key)
update_sp_files(settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id, {'name': data['name'],
'deprecated': data['deprecated'],
'default': data['default']})
copyfile(tmp_policy_file, deploy_policy_file)
rsync_dir_with_nodes(settings.SWIFT_CFG_DEPLOY_DIR, '/etc/swift')
r.hset(key, 'deployed', 'True')
return JSONResponse('Storage policy deployed correctly', status=status.HTTP_200_OK)
except RedisError:
return JSONResponse('Storage policy could not be deployed', status=status.HTTP_400_BAD_REQUEST)
except exceptions.RingBuilderError, e:
return JSONResponse('Storage policy could not be deployed. Error message: %s' % e.message, status=status.HTTP_400_BAD_REQUEST)
else:
return JSONResponse('Storage policy not found.', status=status.HTTP_404_NOT_FOUND)
def test_add_device(self):
self.create_sample_ring()
argv = ["", self.tmpfile, "add",
"r2z3-127.0.0.1:6000/sda3_some meta data", "3.14159265359"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
# Check that device was created with given data
ring = RingBuilder.load(self.tmpfile)
dev = [d for d in ring.devs if d['id'] == 2][0]
self.assertEqual(dev['region'], 2)
self.assertEqual(dev['zone'], 3)
self.assertEqual(dev['ip'], '127.0.0.1')
self.assertEqual(dev['port'], 6000)
self.assertEqual(dev['device'], 'sda3')
self.assertEqual(dev['weight'], 3.14159265359)
self.assertEqual(dev['replication_ip'], '127.0.0.1')
self.assertEqual(dev['replication_port'], 6000)
self.assertEqual(dev['meta'], 'some meta data')
# Final check, rebalance and check ring is ok
ring.rebalance()
self.assertTrue(ring.validate())
def test_add_device(self):
self.create_sample_ring()
argv = ["", self.tmpfile, "add",
"r2z3-127.0.0.1:6000/sda3_some meta data", "3.14159265359"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
# Check that device was created with given data
ring = RingBuilder.load(self.tmpfile)
dev = [d for d in ring.devs if d['id'] == 2][0]
self.assertEqual(dev['region'], 2)
self.assertEqual(dev['zone'], 3)
self.assertEqual(dev['ip'], '127.0.0.1')
self.assertEqual(dev['port'], 6000)
self.assertEqual(dev['device'], 'sda3')
self.assertEqual(dev['weight'], 3.14159265359)
self.assertEqual(dev['replication_ip'], '127.0.0.1')
self.assertEqual(dev['replication_port'], 6000)
self.assertEqual(dev['meta'], 'some meta data')
# Final check, rebalance and check ring is ok
ring.rebalance()
self.assertTrue(ring.validate())
def main(argv):
setup_logging()
args = read_args()
logger.debug(args)
load_config()
v1 = client.CoreV1Api()
config_map = load_config_map(v1, args.config_map_name)
if config_map.binary_data != None and args.ring_type in config_map.binary_data:
builder_ = base64.b64decode(config_map.binary_data[args.ring_type])
logger.info("loading existing ring builder")
builder = RingBuilder.load("", open=lambda a, b: io.BytesIO(builder_))
else:
logger.info("creating a new ring builder")
builder = RingBuilder(10, 1, 1)
r = RingController(builder, args.ring_type, logger=logger)
logger.info("reconciling ring")
r.reconcile(args.devices)
ring_data = r.get_ring_data()
patch_config_map(v1, config_map.metadata, ring_data)
def test_get_ring_data(self):
# given a ring builder with two devices
b = RingBuilder(10, 1, 1)
r = RingController(b, "object")
ds = ["r1z1-192.168.0.2:6000/d3", "r1z2-192.168.2.2:5000/d1"]
r.reconcile(ds)
# when ring is serialized
ring_data = r.get_ring_data()
# then ring and builder is returned
self.assertIn("object.ring.gz", ring_data)
self.assertIn("object", ring_data)
# then correct compressed ring is returned
object_ring = decompress(base64.b64decode(ring_data["object.ring.gz"]))
expected_ring = b'{"byteorder": "little", "devs": [{"device": "d3", "id": 0, "ip": "192.168.0.2", "meta": "", "port": 6000, "region": 1, "replication_ip": null, "replication_port": null, "weight": 1.0, "zone": 1}, {"device": "d1", "id": 1, "ip": "192.168.2.2", "meta": "", "port": 5000, "region": 1, "replication_ip": null, "replication_port": null, "weight": 1.0, "zone": 2}], "part_shift": 22, "replica_count": 1, "version": 3}'
self.assertIn(expected_ring, object_ring)
# then correct serialized builder is returned
builder_ = base64.b64decode(ring_data["object"])
loadedBuilder = RingBuilder.load(
"", open=lambda a, b: io.BytesIO(builder_))
self.assertEqual(b.devs, loadedBuilder.devs)
def main(arguments=None):
global argv, backup_dir, builder, builder_file, ring_file
if arguments is not None:
argv = arguments
else:
argv = sys_argv
if len(argv) < 2:
print("swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" % globals())
print(Commands.default.__doc__.strip())
print()
cmds = [c for c in dir(Commands) if getattr(Commands, c).__doc__ and not c.startswith("_") and c != "default"]
cmds.sort()
for cmd in cmds:
print(getattr(Commands, cmd).__doc__.strip())
print()
print(parse_search_value.__doc__.strip())
print()
for line in wrap(" ".join(cmds), 79, initial_indent="Quick list: ", subsequent_indent=" "):
print(line)
print(
"Exit codes: 0 = operation successful\n"
" 1 = operation completed with warnings\n"
" 2 = error"
)
exit(EXIT_SUCCESS)
builder_file, ring_file = parse_builder_ring_filename_args(argv)
if builder_file != argv[1]:
print("Note: using %s instead of %s as builder file" % (builder_file, argv[1]))
try:
builder = RingBuilder.load(builder_file)
except exceptions.UnPicklingError as e:
print(e)
exit(EXIT_ERROR)
except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:
if len(argv) < 3 or argv[2] not in ("create", "write_builder"):
print(e)
exit(EXIT_ERROR)
except Exception as e:
print("Problem occurred while reading builder file: %s. %s" % (builder_file, e))
exit(EXIT_ERROR)
backup_dir = pathjoin(dirname(builder_file), "backups")
try:
mkdir(backup_dir)
except OSError as err:
if err.errno != EEXIST:
raise
if len(argv) == 2:
command = "default"
else:
command = argv[2]
if argv[0].endswith("-safe"):
try:
with lock_parent_directory(abspath(builder_file), 15):
getattr(Commands, command, Commands.unknown)()
except exceptions.LockTimeout:
print("Ring/builder dir currently locked.")
exit(2)
else:
getattr(Commands, command, Commands.unknown)()
def test_set_min_part_hours(self):
self.create_sample_ring()
argv = ["", self.tmpfile, "set_min_part_hours", "24"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.min_part_hours, 24)
def main(arguments=None):
global argv, backup_dir, builder, builder_file, ring_file
if arguments:
argv = arguments
else:
argv = sys_argv
if len(argv) < 2:
print("swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" %
globals())
print(Commands.default.__doc__.strip())
print()
cmds = [c for c, f in Commands.__dict__.items()
if f.__doc__ and not c.startswith('_') and c != 'default']
cmds.sort()
for cmd in cmds:
print(Commands.__dict__[cmd].__doc__.strip())
print()
print(parse_search_value.__doc__.strip())
print()
for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ',
subsequent_indent=' '):
print(line)
print('Exit codes: 0 = operation successful\n'
' 1 = operation completed with warnings\n'
' 2 = error')
exit(EXIT_SUCCESS)
builder_file, ring_file = parse_builder_ring_filename_args(argv)
if builder_file != argv[1]:
print('Note: using %s instead of %s as builder file' % (
builder_file, argv[1]))
try:
builder = RingBuilder.load(builder_file)
except exceptions.UnPicklingError as e:
print(e)
exit(EXIT_ERROR)
except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:
if len(argv) < 3 or argv[2] not in('create', 'write_builder'):
print(e)
exit(EXIT_ERROR)
except Exception as e:
print('Problem occurred while reading builder file: %s. %s' %
(builder_file, e))
exit(EXIT_ERROR)
backup_dir = pathjoin(dirname(builder_file), 'backups')
try:
mkdir(backup_dir)
except OSError as err:
if err.errno != EEXIST:
raise
if len(argv) == 2:
command = "default"
else:
command = argv[2]
if argv[0].endswith('-safe'):
try:
with lock_parent_directory(abspath(builder_file), 15):
Commands.__dict__.get(command, Commands.unknown.__func__)()
except exceptions.LockTimeout:
print("Ring/builder dir currently locked.")
exit(2)
else:
Commands.__dict__.get(command, Commands.unknown.__func__)()
def main(arguments=None):
global argv, backup_dir, builder, builder_file, ring_file
if arguments:
argv = arguments
else:
argv = sys_argv
if len(argv) < 2:
print("swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" %
globals())
print(Commands.default.__doc__.strip())
print()
cmds = [
c for c, f in Commands.__dict__.items()
if f.__doc__ and c[0] != '_' and c != 'default'
]
cmds.sort()
for cmd in cmds:
print(Commands.__dict__[cmd].__doc__.strip())
print()
print(parse_search_value.__doc__.strip())
print()
for line in wrap(' '.join(cmds),
79,
initial_indent='Quick list: ',
subsequent_indent=' '):
print(line)
print('Exit codes: 0 = operation successful\n'
' 1 = operation completed with warnings\n'
' 2 = error')
exit(EXIT_SUCCESS)
builder_file, ring_file = parse_builder_ring_filename_args(argv)
if builder_file != argv[1]:
print('Note: using %s instead of %s as builder file' %
(builder_file, argv[1]))
try:
builder = RingBuilder.load(builder_file)
except exceptions.UnPicklingError as e:
print(e)
exit(EXIT_ERROR)
except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:
if len(argv) < 3 or argv[2] not in ('create', 'write_builder'):
print(e)
exit(EXIT_ERROR)
except Exception as e:
print('Problem occurred while reading builder file: %s. %s' %
(builder_file, e))
exit(EXIT_ERROR)
backup_dir = pathjoin(dirname(builder_file), 'backups')
try:
mkdir(backup_dir)
except OSError as err:
if err.errno != EEXIST:
raise
if len(argv) == 2:
command = "default"
else:
command = argv[2]
if argv[0].endswith('-safe'):
try:
with lock_parent_directory(abspath(builder_file), 15):
Commands.__dict__.get(command, Commands.unknown.__func__)()
except exceptions.LockTimeout:
print("Ring/builder dir currently locked.")
exit(2)
else:
Commands.__dict__.get(command, Commands.unknown.__func__)()
def load_swift_policies(request):
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
if request.method == "POST":
# 1st step: Copy de swift.conf file from a Proxy Server to a local Crystal directory
proxy_nodes = r.keys("proxy_node:*")
if proxy_nodes:
node = r.hgetall(proxy_nodes[0])
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(node['ip'],
username=node['ssh_username'],
password=node['ssh_password'])
try:
sftp_client = ssh_client.open_sftp()
swift_etc_path = '/etc/swift/'
remote_swift_file = swift_etc_path + 'swift.conf'
local_swift_file_deploy = get_swift_cfg_path(
settings.SWIFT_CFG_DEPLOY_DIR)
sftp_client.get(remote_swift_file, local_swift_file_deploy)
remote_file_list = sftp_client.listdir(swift_etc_path)
for r_file in remote_file_list:
if r_file.startswith('object') and r_file.endswith(
'.builder'):
remote_file = swift_etc_path + r_file
local_file_tmp = os.path.join(
settings.SWIFT_CFG_TMP_DIR, r_file)
local_file_deploy = os.path.join(
settings.SWIFT_CFG_DEPLOY_DIR, r_file)
sftp_client.get(remote_file, local_file_tmp)
sftp_client.get(remote_file, local_file_deploy)
except SSHException:
ssh_client.close()
logger.error('An error occurred restarting Swift nodes')
raise FileSynchronizationException(
"An error occurred restarting Swift nodes")
sftp_client.close()
ssh_client.close()
# 2nd step: load policies
pattern = os.path.join(settings.SWIFT_CFG_TMP_DIR, 'object*')
files = glob.glob(pattern)
try:
for builder_file in files:
builder = RingBuilder.load(builder_file)
if '-' in builder_file:
sp_id = builder_file.split('.')[0].split('-')[-1]
key = 'storage-policy:' + sp_id
if int(sp_id) > r.get('storage-policies:id'):
r.set('storage-policies:id', sp_id)
else:
key = 'storage-policy:0'
local_swift_file = get_swift_cfg_path(
settings.SWIFT_CFG_DEPLOY_DIR)
config_parser = ConfigParser.RawConfigParser()
config_parser.read(local_swift_file)
if config_parser.has_section(key):
name = config_parser.get(
key, 'name') if config_parser.has_option(
key, 'name') else 'Policy-' + sp_id
policy_type = config_parser.get(
key, 'policy_type') if config_parser.has_option(
key, 'policy_type') else 'Replication'
deprecated = config_parser.get(
key, 'deprecated') if config_parser.has_option(
key, 'deprecated') else 'False'
if config_parser.has_option(key, 'default'):
default = 'True' if config_parser.get(
key, 'default') in ['yes', 'Yes'] else 'False'
else:
default = 'False'
devices = []
nodes_data = {}
nodes = r.keys('*_node:*')
for node in nodes:
nodes_data[node] = r.hgetall(node)
for device in builder.devs:
try:
inet_aton(device['ip'])
device['ip'] = next(
(nodes_data[node]['name']
for node in nodes_data
if nodes_data[node]['ip'] == device['ip']),
device['ip'])
except:
pass
devices.append((device['ip'] + ':' + device['device'],
device['id']))
data = {
'name': name,
'default': default,
'deprecated': deprecated,
'time': builder.min_part_hours,
'devices': json.dumps(devices),
'deployed': 'True',
'policy_type':
policy_type if policy_type else 'Replication',
'partition_power': int(math.log(builder.parts, 2)),
'replicas': int(builder.replicas)
}
r.hmset(key, data)
else:
pass
except RedisError:
return JSONResponse('Policies could not be loaded',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return JSONResponse('Policies loaded correctly',
status=status.HTTP_200_OK)
return JSONResponse('Only HTTP POST requests allowed.',
status=status.HTTP_405_METHOD_NOT_ALLOWED)
def test_set_min_part_hours(self):
self.create_sample_ring()
argv = ["", self.tmpfile, "set_min_part_hours", "24"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.min_part_hours, 24)
def test_set_replicas(self):
self.create_sample_ring()
argv = ["", self.tmpfile, "set_replicas", "3.14159265359"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.replicas, 3.14159265359)
def storage_policy_disks(request, storage_policy_id):
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
key = "storage-policy:" + storage_policy_id
if request.method == 'GET':
if r.exists(key):
storage_policy = r.hgetall(key)
storage_policy['devices'] = json.loads(storage_policy['devices'])
all_devices = []
for node_key in r.keys('object_node:*'):
node = r.hgetall(node_key)
all_devices += [
node_key.split(':')[1] + ':' + device
for device in json.loads(node['devices']).keys()
]
current_devices = [dev[0] for dev in storage_policy['devices']]
available_devices = [
device for device in all_devices
if device not in current_devices
]
available_devices_detail = []
for device in available_devices:
object_node_id, device_id = device.split(':')
object_node = r.hgetall('object_node:' + object_node_id)
device_detail = json.loads(object_node['devices'])[device_id]
device_detail['id'] = device
device_detail['region'] = r.hgetall(
'region:' + object_node['region_id'])['name']
device_detail['zone'] = r.hgetall(
'zone:' + object_node['zone_id'])['name']
available_devices_detail.append(device_detail)
return JSONResponse(available_devices_detail,
status=status.HTTP_200_OK)
else:
return JSONResponse('Storage policy not found.',
status=status.HTTP_404_NOT_FOUND)
if request.method == 'PUT':
if r.exists(key):
disk = JSONParser().parse(request)
object_node_id, device_id = disk.split(':')
object_node = r.hgetall('object_node:' + object_node_id)
# device_detail = json.loads(object_node['devices'])[device_id]
region = r.hgetall('region:' + object_node['region_id'])['name']
zone = r.hgetall('zone:' + object_node['zone_id'])['name']
tmp_policy_file = get_policy_file_path(settings.SWIFT_CFG_TMP_DIR,
storage_policy_id)
ring = RingBuilder.load(tmp_policy_file)
ring_dev_id = ring.add_dev({
'weight': 100,
'region': region,
'zone': zone,
'ip': object_node['ip'],
'port': '6200',
'device': device_id
})
ring.save(tmp_policy_file)
storage_policy = r.hgetall(key)
storage_policy['devices'] = json.loads(storage_policy['devices'])
storage_policy['devices'].append((disk, ring_dev_id))
storage_policy['devices'] = json.dumps(storage_policy['devices'])
r.hset(key, 'devices', storage_policy['devices'])
r.hset(key, 'deployed', False)
return JSONResponse('Disk added correctly',
status=status.HTTP_200_OK)
else:
return JSONResponse('Disk could not be added.',
status=status.HTTP_400_BAD_REQUEST)
return JSONResponse('Method not allowed.',
status=status.HTTP_405_METHOD_NOT_ALLOWED)
def test_set_overload_negative(self):
self.create_sample_ring()
argv = ["", self.tmpfile, "set_overload", "-0.19878"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.overload, 0.0)
def load_swift_policies(request):
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
if request.method == "POST":
# 1st step: Copy de swift.conf file from a Proxy Server to a local Crystal directory
proxy_nodes = r.keys("proxy_node:*")
if proxy_nodes:
node = r.hgetall(proxy_nodes[0])
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(node['ip'], username=node['ssh_username'], password=node['ssh_password'])
try:
sftp_client = ssh_client.open_sftp()
swift_etc_path = '/etc/swift/'
remote_swift_file = swift_etc_path+'swift.conf'
local_swift_file_deploy = get_swift_cfg_path(settings.SWIFT_CFG_DEPLOY_DIR)
sftp_client.get(remote_swift_file, local_swift_file_deploy)
remote_file_list = sftp_client.listdir(swift_etc_path)
for r_file in remote_file_list:
if r_file.startswith('object') and r_file.endswith('.builder'):
remote_file = swift_etc_path+r_file
local_file_tmp = os.path.join(settings.SWIFT_CFG_TMP_DIR, r_file)
local_file_deploy = os.path.join(settings.SWIFT_CFG_DEPLOY_DIR, r_file)
sftp_client.get(remote_file, local_file_tmp)
sftp_client.get(remote_file, local_file_deploy)
except SSHException:
ssh_client.close()
logger.error('An error occurred restarting Swift nodes')
raise FileSynchronizationException("An error occurred restarting Swift nodes")
sftp_client.close()
ssh_client.close()
# 2nd step: load policies
pattern = os.path.join(settings.SWIFT_CFG_TMP_DIR, 'object*')
files = glob.glob(pattern)
try:
for builder_file in files:
builder = RingBuilder.load(builder_file)
if '-' in builder_file:
sp_id = builder_file.split('.')[0].split('-')[-1]
key = 'storage-policy:' + sp_id
if int(sp_id) > r.get('storage-policies:id'):
r.set('storage-policies:id', sp_id)
else:
key = 'storage-policy:0'
local_swift_file = get_swift_cfg_path(settings.SWIFT_CFG_DEPLOY_DIR)
config_parser = ConfigParser.RawConfigParser()
config_parser.read(local_swift_file)
if config_parser.has_section(key):
name = config_parser.get(key, 'name') if config_parser.has_option(key, 'name') else 'Policy-' + sp_id
policy_type = config_parser.get(key, 'policy_type') if config_parser.has_option(key, 'policy_type') else 'Replication'
deprecated = config_parser.get(key, 'deprecated') if config_parser.has_option(key, 'deprecated') else 'False'
if config_parser.has_option(key, 'default'):
default = 'True' if config_parser.get(key, 'default') in ['yes', 'Yes'] else 'False'
else:
default = 'False'
devices = []
nodes_data = {}
nodes = r.keys('*_node:*')
for node in nodes:
nodes_data[node] = r.hgetall(node)
for device in builder.devs:
try:
inet_aton(device['ip'])
device['ip'] = next((nodes_data[node]['name'] for node in nodes_data if nodes_data[node]['ip'] == device['ip']), device['ip'])
except:
pass
devices.append((device['ip'] + ':' + device['device'], device['id']))
data = {'name': name,
'default': default,
'deprecated': deprecated,
'time': builder.min_part_hours,
'devices': json.dumps(devices),
'deployed': 'True',
'policy_type': policy_type if policy_type else 'Replication',
'partition_power': int(math.log(builder.parts, 2)),
'replicas': int(builder.replicas)
}
r.hmset(key, data)
else:
pass
except RedisError:
return JSONResponse('Policies could not be loaded', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return JSONResponse('Policies loaded correctly', status=status.HTTP_200_OK)
return JSONResponse('Only HTTP POST requests allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def test_set_replicas(self):
self.create_sample_ring()
argv = ["", self.tmpfile, "set_replicas", "3.14159265359"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.replicas, 3.14159265359)
def test_set_overload_non_numeric(self):
self.create_sample_ring()
argv = ["", self.tmpfile, "set_overload", "swedish fish"]
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
ring = RingBuilder.load(self.tmpfile)
self.assertEqual(ring.overload, 0.0)
