def create():
"""
swift-ring-builder <builder_file> create <part_power> <replicas>
<min_part_hours>
Creates <builder_file> with 2^<part_power> partitions and <replicas>.
<min_part_hours> is number of hours to restrict moving a partition more
than once.
"""
if len(argv) < 6:
print(Commands.create.__doc__.strip())
exit(EXIT_ERROR)
# 1、生成RingBuilder对象实例
builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]))
# 2、创建备份目录
backup_dir = pathjoin(dirname(builder_file), 'backups')
try:
mkdir(backup_dir)
except OSError as err:
if err.errno != EEXIST:
raise
# 3、保存原始数据到备份目录，以及/etc/swift目录中
builder.save(pathjoin(backup_dir,
'%d.' % time() + basename(builder_file)))
builder.save(builder_file)
exit(EXIT_SUCCESS)
def test_compose_rings_float_replica_count_builder_error(self):
builders = self.create_sample_ringbuilders(1)
# prepare another ring which has float replica count
incorrect_builder = RingBuilder(6, 1.5, 1)
_, fname = tempfile.mkstemp(dir=self.tmpdir)
for _ in range(4):
dev = self.pop_region_device(1)
incorrect_builder.add_dev(dev)
incorrect_builder.rebalance()
incorrect_builder.save(fname)
# sanity
self.assertTrue(os.path.exists(fname))
self.assertEqual(1.5, incorrect_builder.replicas)
# the first replica has 2 ** 6 partitions
self.assertEqual(
2 ** 6, len(incorrect_builder._replica2part2dev[0]))
# but the second replica has the half of the first partitions
self.assertEqual(
2 ** 5, len(incorrect_builder._replica2part2dev[1]))
builders.append(incorrect_builder)
with self.assertRaises(ValueError) as cm:
compose_rings(builders)
self.assertIn("Problem with builders", cm.exception.message)
self.assertIn("Non integer replica count", cm.exception.message)
def create_sample_ring(self):
""" Create a sample ring with two devices
At least two devices are needed to test removing
a device, since removing the last device of a ring
is not allowed """
# Ensure there is no existing test builder file because
# create_sample_ring() might be used more than once in a single test
try:
os.remove(self.tmpfile)
except OSError:
pass
ring = RingBuilder(6, 3, 1)
ring.add_dev({'weight': 100.0,
'region': 0,
'zone': 0,
'ip': '127.0.0.1',
'port': 6000,
'device': 'sda1',
'meta': 'some meta data',
})
ring.add_dev({'weight': 100.0,
'region': 1,
'zone': 1,
'ip': '127.0.0.2',
'port': 6001,
'device': 'sda2'
})
ring.save(self.tmpfile)
def create_sample_ring(self):
""" Create a sample ring with two devices
At least two devices are needed to test removing
a device, since removing the last device of a ring
is not allowed """
# Ensure there is no existing test builder file because
# create_sample_ring() might be used more than once in a single test
try:
os.remove(self.tmpfile)
except OSError:
pass
ring = RingBuilder(6, 3, 1)
ring.add_dev({'weight': 100.0,
'region': 0,
'zone': 0,
'ip': '127.0.0.1',
'port': 6000,
'device': 'sda1',
'meta': 'some meta data',
})
ring.add_dev({'weight': 100.0,
'region': 1,
'zone': 1,
'ip': '127.0.0.2',
'port': 6001,
'device': 'sda2'
})
ring.save(self.tmpfile)
def create():
"""
swift-ring-builder <builder_file> create <part_power> <replicas>
<min_part_hours>
Creates <builder_file> with 2^<part_power> partitions and <replicas>.
<min_part_hours> is number of hours to restrict moving a partition more
than once.
OR
swift-ring-builder <builder_file> create <part_power> <replicas>
<min_part_hours> <policy_info_file>
policy_info_file contains the information about the policy
"""
if len(argv) < 6:
print Commands.create.__doc__.strip()
exit(EXIT_ERROR)
if len(argv) > 6:
builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]),
argv[6])
else:
builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]))
backup_dir = pathjoin(dirname(argv[1]), 'backups')
try:
mkdir(backup_dir)
except OSError as err:
if err.errno != EEXIST:
raise
builder.save(pathjoin(backup_dir, '%d.' % time() + basename(argv[1])))
builder.save(argv[1])
exit(EXIT_SUCCESS)
def create_sample_ringbuilders(self, num_builders=2):
"""
Create sample rings with four devices
:returns: a list of ring builder instances
"""
builders = []
for region in range(num_builders):
fname = os.path.join(self.tmpdir, 'builder_%s.builder' % region)
builder = RingBuilder(6, 3, 0)
for _ in range(5):
dev = self.pop_region_device(region)
builder.add_dev(dev)
# remove last dev to simulate a ring with some history
builder.remove_dev(dev['id'])
# add a dev that won't be assigned any parts
new_dev = self.pop_region_device(region)
new_dev['weight'] = 0
builder.add_dev(new_dev)
builder.rebalance()
builder.save(fname)
self.assertTrue(os.path.exists(fname))
builders.append(builder)
return builders
def create():
"""
swift-ring-builder <builder_file> create <part_power> <replicas>
<min_part_hours>
Creates <builder_file> with 2^<part_power> partitions and <replicas>.
<min_part_hours> is number of hours to restrict moving a partition more
than once.
OR
swift-ring-builder <builder_file> create <part_power> <replicas>
<min_part_hours> <policy_info_file>
policy_info_file contains the information about the policy
"""
if len(argv) < 6:
print Commands.create.__doc__.strip()
exit(EXIT_ERROR)
if len (argv) > 6:
builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]), argv[6])
else:
builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]))
backup_dir = pathjoin(dirname(argv[1]), 'backups')
try:
mkdir(backup_dir)
except OSError as err:
if err.errno != EEXIST:
raise
builder.save(pathjoin(backup_dir, '%d.' % time() + basename(argv[1])))
builder.save(argv[1])
exit(EXIT_SUCCESS)
def create():
"""
swift-ring-builder <builder_file> create <part_power> <replicas>
<min_part_hours>
Creates <builder_file> with 2^<part_power> partitions and <replicas>.
<min_part_hours> is number of hours to restrict moving a partition more
than once.
"""
if len(argv) < 6:
print(Commands.create.__doc__.strip())
exit(EXIT_ERROR)
# 1、生成RingBuilder对象实例
builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]))
# 2、创建备份目录
backup_dir = pathjoin(dirname(builder_file), 'backups')
try:
mkdir(backup_dir)
except OSError as err:
if err.errno != EEXIST:
raise
# 3、保存原始数据到备份目录，以及/etc/swift目录中
builder.save(
pathjoin(backup_dir, '%d.' % time() + basename(builder_file)))
builder.save(builder_file)
exit(EXIT_SUCCESS)
def write_stub_builder(tmpdir, region=1, name=''):
"""
Pretty much just a three node, three replica, 8 part power builder...
:param tmpdir: a place to write the builder, be sure to clean it up!
:param region: an integer, fills in region and ip
:param name: the name of the builder (i.e. <name>.builder)
"""
name = name or str(region)
replicas = 3
builder = RingBuilder(8, replicas, 1)
for i in range(replicas):
dev = {
'weight': 100,
'region': '%d' % region,
'zone': '1',
'ip': '10.0.0.%d' % region,
'port': '3600',
'device': 'sdb%d' % i
}
builder.add_dev(dev)
builder.rebalance()
builder_file = os.path.join(tmpdir, '%s.builder' % name)
builder.save(builder_file)
return builder, builder_file
def test_different_part_power_error(self):
# create a ring builder
# (default, part power is 6 with create_sample_ringbuilders)
builders = self.create_sample_ringbuilders(1)
# prepare another ring which has different part power
incorrect_builder = RingBuilder(4, 3, 1)
_, fname = tempfile.mkstemp(dir=self.tmpdir)
for _ in range(4):
dev = self.pop_region_device(1)
incorrect_builder.add_dev(dev)
incorrect_builder.rebalance()
incorrect_builder.save(fname)
# sanity
self.assertTrue(os.path.exists(fname))
# sanity
correct_builder = builders[0]
self.assertNotEqual(correct_builder.part_shift,
incorrect_builder.part_shift)
self.assertNotEqual(correct_builder.part_power,
incorrect_builder.part_power)
builders.append(incorrect_builder)
with self.assertRaises(ValueError) as cm:
compose_rings(builders)
self.assertIn("All builders must have same value for 'part_power'",
cm.exception.message)
def test_composite_same_device_in_the_different_rings_error(self):
builders = self.create_sample_ringbuilders(2)
same_device = copy.deepcopy(builders[0].devs[0])
# create one more ring which duplicates a device in the first ring
builder = RingBuilder(6, 3, 1)
_, fname = tempfile.mkstemp(dir=self.tmpdir)
# add info to feed to add_dev
same_device.update({'region': 2, 'weight': 100})
builder.add_dev(same_device)
# add rest of the devices, which are unique
for _ in range(3):
dev = self.pop_region_device(2)
builder.add_dev(dev)
builder.rebalance()
builder.save(fname)
# sanity
self.assertTrue(os.path.exists(fname))
builders.append(builder)
with self.assertRaises(ValueError) as cm:
compose_rings(builders)
self.assertIn(
'Duplicate ip/port/device combination %(ip)s/%(port)s/%(device)s '
'found in builders at indexes 0 and 2' % same_device,
cm.exception.message)
def create_sample_ring(self):
""" Create a sample ring with two devices
At least two devices are needed to test removing
a device, since removing the last device of a ring
is not allowed """
# Ensure there is no existing test builder file because
# create_sample_ring() might be used more than once in a single test
try:
os.remove(self.tmpfile)
except OSError:
pass
ring = RingBuilder(6, 3, 1)
ring.add_dev(
{
"weight": 100.0,
"region": 0,
"zone": 0,
"ip": "127.0.0.1",
"port": 6000,
"device": "sda1",
"meta": "some meta data",
}
)
ring.add_dev({"weight": 100.0, "region": 1, "zone": 1, "ip": "127.0.0.2", "port": 6001, "device": "sda2"})
ring.save(self.tmpfile)
def storage_policies(request):
"""
Creates a storage policy to swift with an specific ring.
Allows create replication storage policies and erasure code storage policies
"""
if request.method == "GET":
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
keys = r.keys("storage-policy:*")
storage_policy_list = []
for key in keys:
storage_policy = r.hgetall(key)
to_json_bools(storage_policy, 'deprecated', 'default', 'deployed')
storage_policy['id'] = str(key).split(':')[-1]
storage_policy['devices'] = json.loads(storage_policy['devices'])
storage_policy_list.append(storage_policy)
return JSONResponse(storage_policy_list, status=status.HTTP_200_OK)
if request.method == "POST":
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
data = JSONParser().parse(request)
if data['policy_type'] == 'EC':
data['replicas'] = int(data['ec_num_data_fragments']) + int(
data['ec_num_parity_fragments'])
try:
sp_id = str(r.incr('storage-policies:id'))
key = 'storage-policy:' + sp_id
ring = RingBuilder(int(data['partition_power']),
int(data['replicas']), int(data['time']))
ring.save(get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, sp_id))
r.hmset(key, data)
except:
return JSONResponse('Error creating the Storage Policy',
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return JSONResponse('Account created successfully',
status=status.HTTP_201_CREATED)
return JSONResponse('Only HTTP POST requests allowed.',
status=status.HTTP_405_METHOD_NOT_ALLOWED)
def write_ring(args, devices, builderfile):
# Make an educated guess about the used port. These are the defaults for
# TripleO-based deployments in Mitaka
builder_fname = os.path.basename(builderfile)
if 'account' in builder_fname:
port = 6002
elif 'container' in builder_fname:
port = 6001
elif 'object' in builder_fname:
port = 6000
else:
port = 6000
logging.debug('Set port for new devices to %d' % port)
if not os.path.isfile(builderfile):
logging.info(
'%s not found, creating new builder file', builderfile)
rb = RingBuilder(args.part_power, args.replicas, args.min_part_hours)
else:
logging.info('Using existing builder file %s', builderfile)
rb = RingBuilder.load(builderfile)
# Add all missing devices
for dev in devices:
_dev = rb.search_devs({'meta': dev['meta']})
if not _dev:
dev['weight'] = float(dev.get('size')) / 10**9
dev['region'] = 1
dev['zone'] = 1
dev['port'] = port
dev['replication_ip'] = dev['ip']
dev['replication_port'] = dev['port']
rb.add_dev(dev)
logging.info('Added device %s / %s', dev['ip'], dev['device'])
else:
logging.info(
'Ignoring existing device %s / %s', dev['ip'], dev['device'])
try:
rb.rebalance()
except RingValidationError as exc:
logging.error(exc)
rb.save(builderfile)
ring_file = os.path.splitext(builderfile)[0] + '.ring.gz'
ring_data = rb.get_ring()
ring_data.save(ring_file)
return [builderfile, ring_file]
def storage_policies(request):
"""
Creates a storage policy to swift with an specific ring.
Allows create replication storage policies and erasure code storage policies
"""
if request.method == "GET":
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
keys = r.keys("storage-policy:*")
storage_policy_list = []
for key in keys:
storage_policy = r.hgetall(key)
to_json_bools(storage_policy, 'deprecated', 'default', 'deployed')
storage_policy['id'] = str(key).split(':')[-1]
storage_policy['devices'] = json.loads(storage_policy['devices'])
storage_policy_list.append(storage_policy)
return JSONResponse(storage_policy_list, status=status.HTTP_200_OK)
if request.method == "POST":
try:
r = get_redis_connection()
except RedisError:
return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
data = JSONParser().parse(request)
if data['policy_type'] == 'EC':
data['replicas'] = int(data['ec_num_data_fragments']) + int(data['ec_num_parity_fragments'])
try:
sp_id = str(r.incr('storage-policies:id'))
key = 'storage-policy:' + sp_id
ring = RingBuilder(int(data['partition_power']), int(data['replicas']), int(data['time']))
ring.save(get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, sp_id))
r.hmset(key, data)
except:
return JSONResponse('Error creating the Storage Policy', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
return JSONResponse('Account created successfully', status=status.HTTP_201_CREATED)
return JSONResponse('Only HTTP POST requests allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def _write_stub_builder(self, region):
replicas = 3
builder = RingBuilder(8, replicas, 1)
for i in range(replicas):
dev = {
'weight': 100,
'region': '%d' % region,
'zone': '1',
'ip': '10.0.0.%d' % region,
'port': '3600',
'device': 'sdb%d' % i
}
builder.add_dev(dev)
builder.rebalance()
builder_file = os.path.join(self.tmpdir, '%d.builder' % region)
builder.save(builder_file)
return builder, builder_file
def write_ring(args, devices, builderfile):
# Make an educated guess about the used port. These are the defaults for
# TripleO-based deployments in Mitaka
builder_fname = os.path.basename(builderfile)
if 'account' in builder_fname:
port = 6002
elif 'container' in builder_fname:
port = 6001
elif 'object' in builder_fname:
port = 6000
else:
port = 6000
logging.debug('Set port for new devices to %d' % port)
if not os.path.isfile(builderfile):
logging.info('%s not found, creating new builder file', builderfile)
rb = RingBuilder(args.part_power, args.replicas, args.min_part_hours)
else:
logging.info('Using existing builder file %s', builderfile)
rb = RingBuilder.load(builderfile)
# Add all missing devices
for dev in devices:
_dev = rb.search_devs({'meta': dev['meta']})
if not _dev:
dev['weight'] = float(dev.get('size')) / 10**9
dev['region'] = 1
dev['zone'] = 1
dev['port'] = port
dev['replication_ip'] = dev['ip']
dev['replication_port'] = dev['port']
rb.add_dev(dev)
logging.info('Added device %s / %s', dev['ip'], dev['device'])
else:
logging.info('Ignoring existing device %s / %s', dev['ip'],
dev['device'])
try:
rb.rebalance()
except RingValidationError as exc:
logging.error(exc)
rb.save(builderfile)
ring_file = os.path.splitext(builderfile)[0] + '.ring.gz'
ring_data = rb.get_ring()
ring_data.save(ring_file)
return [builderfile, ring_file]
def write_builder():
"""
swift-ring-builder <ring_file> write_builder [min_part_hours]
Recreate a builder from a ring file (lossy) if you lost your builder
backups. (Protip: don't lose your builder backups).
[min_part_hours] is one of those numbers lost to the builder,
you can change it with set_min_part_hours.
"""
if exists(builder_file):
print 'Cowardly refusing to overwrite existing ' \
'Ring Builder file: %s' % builder_file
exit(EXIT_ERROR)
if len(argv) > 3:
min_part_hours = int(argv[3])
else:
stderr.write("WARNING: default min_part_hours may not match "
"the value in the lost builder.\n")
min_part_hours = 24
ring = Ring(ring_file)
for dev in ring.devs:
dev.update({
'parts': 0,
'parts_wanted': 0,
})
builder_dict = {
'part_power': 32 - ring._part_shift,
'replicas': float(ring.replica_count),
'min_part_hours': min_part_hours,
'parts': ring.partition_count,
'devs': ring.devs,
'devs_changed': False,
'version': 0,
'_replica2part2dev': ring._replica2part2dev_id,
'_last_part_moves_epoch': None,
'_last_part_moves': None,
'_last_part_gather_start': 0,
'_remove_devs': [],
}
builder = RingBuilder(1, 1, 1)
builder.copy_from(builder_dict)
for parts in builder._replica2part2dev:
for dev_id in parts:
builder.devs[dev_id]['parts'] += 1
builder._set_parts_wanted()
builder.save(builder_file)
def write_builder():
"""
swift-ring-builder <ring_file> write_builder [min_part_hours]
Recreate a builder from a ring file (lossy) if you lost your builder
backups. (Protip: don't lose your builder backups).
[min_part_hours] is one of those numbers lost to the builder,
you can change it with set_min_part_hours.
"""
if exists(builder_file):
print 'Cowardly refusing to overwrite existing ' \
'Ring Builder file: %s' % builder_file
exit(EXIT_ERROR)
if len(argv) > 3:
min_part_hours = int(argv[3])
else:
stderr.write("WARNING: default min_part_hours may not match "
"the value in the lost builder.\n")
min_part_hours = 24
ring = Ring(ring_file)
for dev in ring.devs:
dev.update({
'parts': 0,
'parts_wanted': 0,
})
builder_dict = {
'part_power': 32 - ring._part_shift,
'replicas': float(ring.replica_count),
'min_part_hours': min_part_hours,
'parts': ring.partition_count,
'devs': ring.devs,
'devs_changed': False,
'version': 0,
'_replica2part2dev': ring._replica2part2dev_id,
'_last_part_moves_epoch': None,
'_last_part_moves': None,
'_last_part_gather_start': 0,
'_remove_devs': [],
}
builder = RingBuilder(1, 1, 1)
builder.copy_from(builder_dict)
for parts in builder._replica2part2dev:
for dev_id in parts:
builder.devs[dev_id]['parts'] += 1
builder._set_parts_wanted()
builder.save(builder_file)
def remake_rings(partpower=16, weight=100, replicas=3, min_part_hours=1, drives=False):
for service in ['object', 'container', 'account']:
builder_file = '/etc/swift/%s.builder' % service
ring_file = '/etc/swift/%s.ring.gz' % service
try:
os.remove(builder_file)
os.remove(ring_file)
except OSError:
pass
builder = RingBuilder(partpower, replicas, min_part_hours)
builder.save(builder_file)
devstrs = _get_builder_strings(service, drives)
for devstr in devstrs:
new_dev = _parse_add_values(devstr, str(weight))
builder.add_dev(new_dev[0])
builder.save(builder_file)
builder.rebalance(builder_file)
builder.get_ring().save(ring_file)
def remake_rings(partpower=16,
weight=100,
replicas=3,
min_part_hours=1,
drives=False):
for service in ['object', 'container', 'account']:
builder_file = '/etc/swift/%s.builder' % service
ring_file = '/etc/swift/%s.ring.gz' % service
try:
os.remove(builder_file)
os.remove(ring_file)
except OSError:
pass
builder = RingBuilder(partpower, replicas, min_part_hours)
builder.save(builder_file)
devstrs = _get_builder_strings(service, drives)
for devstr in devstrs:
new_dev = _parse_add_values(devstr, str(weight))
builder.add_dev(new_dev[0])
builder.save(builder_file)
builder.rebalance(builder_file)
builder.get_ring().save(ring_file)
def write_stub_builder(tmpdir, region=1, name=''):
"""
Pretty much just a three node, three replica, 8 part power builder...
:param tmpdir: a place to write the builder, be sure to clean it up!
:param region: an integer, fills in region and ip
:param name: the name of the builder (i.e. <name>.builder)
"""
name = name or str(region)
replicas = 3
builder = RingBuilder(8, replicas, 1)
for i in range(replicas):
dev = {'weight': 100,
'region': '%d' % region,
'zone': '1',
'ip': '10.0.0.%d' % region,
'port': '3600',
'device': 'sdb%d' % i}
builder.add_dev(dev)
builder.rebalance()
builder_file = os.path.join(tmpdir, '%s.builder' % name)
builder.save(builder_file)
return builder, builder_file
def test_different_replica_count_works(self):
# create a ring builder
# (default, part power is 6 with create_sample_ringbuilders)
builders = self.create_sample_ringbuilders(1)
# prepare another ring which has different replica count
builder = RingBuilder(6, 1, 1)
_, fname = tempfile.mkstemp(dir=self.tmpdir)
for _ in range(4):
dev = self.pop_region_device(1)
builder.add_dev(dev)
builder.rebalance()
builder.save(fname)
# sanity
self.assertTrue(os.path.exists(fname))
builders.append(builder)
rd = compose_rings(builders)
rd.save(self.output_ring)
got_ring = Ring(self.output_ring)
self.assertEqual(got_ring.partition_count, 2**6)
self.assertEqual(got_ring.replica_count, 4) # 3 + 1
self.assertEqual(got_ring._part_shift, 26)
self.assertDevices(got_ring, builders)
