def default():
"""
swift-ring-builder <builder_file>
Shows information about the ring and the devices within. Output
includes a table that describes the report parameters (id, region,
port, flags, etc).
flags: possible values are 'DEL' and ''
DEL - indicates that the device is marked for removal from
ring and will be removed in next rebalance.
"""
print('%s, build version %d' % (builder_file, builder.version))
regions = 0
zones = 0
balance = 0
dev_count = 0
if builder.devs:
regions = len(set(d['region'] for d in builder.devs
if d is not None))
zones = len(set((d['region'], d['zone']) for d in builder.devs
if d is not None))
dev_count = len([dev for dev in builder.devs
if dev is not None])
balance = builder.get_balance()
dispersion_trailer = '' if builder.dispersion is None else (
', %.02f dispersion' % (builder.dispersion))
print('%d partitions, %.6f replicas, %d regions, %d zones, '
'%d devices, %.02f balance%s' % (
builder.parts, builder.replicas, regions, zones, dev_count,
balance, dispersion_trailer))
print('The minimum number of hours before a partition can be '
'reassigned is %s (%s remaining)' % (
builder.min_part_hours,
timedelta(seconds=builder.min_part_seconds_left)))
print('The overload factor is %0.2f%% (%.6f)' % (
builder.overload * 100, builder.overload))
# compare ring file against builder file
if not exists(ring_file):
print('Ring file %s not found, '
'probably it hasn\'t been written yet' % ring_file)
else:
builder_dict = builder.get_ring().to_dict()
try:
ring_dict = RingData.load(ring_file).to_dict()
except Exception as exc:
print('Ring file %s is invalid: %r' % (ring_file, exc))
else:
if builder_dict == ring_dict:
print('Ring file %s is up-to-date' % ring_file)
else:
print('Ring file %s is obsolete' % ring_file)
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
header_line, print_dev_f = _make_display_device_table(builder)
print(header_line)
for dev in builder._iter_devs():
flags = 'DEL' if dev in builder._remove_devs else ''
print_dev_f(dev, balance_per_dev[dev['id']], flags)
exit(EXIT_SUCCESS)
def default():
"""
swift-ring-builder <builder_file>
Shows information about the ring and the devices within. Output
includes a table that describes the report parameters (id, region,
port, flags, etc).
flags: possible values are 'DEL' and ''
DEL - indicates that the device is marked for removal from
ring and will be removed in next rebalance.
"""
print('%s, build version %d' % (builder_file, builder.version))
regions = 0
zones = 0
balance = 0
dev_count = 0
if builder.devs:
regions = len(set(d['region'] for d in builder.devs
if d is not None))
zones = len(set((d['region'], d['zone']) for d in builder.devs
if d is not None))
dev_count = len([dev for dev in builder.devs
if dev is not None])
balance = builder.get_balance()
dispersion_trailer = '' if builder.dispersion is None else (
', %.02f dispersion' % (builder.dispersion))
print('%d partitions, %.6f replicas, %d regions, %d zones, '
'%d devices, %.02f balance%s' % (
builder.parts, builder.replicas, regions, zones, dev_count,
balance, dispersion_trailer))
print('The minimum number of hours before a partition can be '
'reassigned is %s (%s remaining)' % (
builder.min_part_hours,
timedelta(seconds=builder.min_part_seconds_left)))
print('The overload factor is %0.2f%% (%.6f)' % (
builder.overload * 100, builder.overload))
# compare ring file against builder file
if not exists(ring_file):
print('Ring file %s not found, '
'probably it hasn\'t been written yet' % ring_file)
else:
builder_dict = builder.get_ring().to_dict()
try:
ring_dict = RingData.load(ring_file).to_dict()
except Exception as exc:
print('Ring file %s is invalid: %r' % (ring_file, exc))
else:
if builder_dict == ring_dict:
print('Ring file %s is up-to-date' % ring_file)
else:
print('Ring file %s is obsolete' % ring_file)
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
header_line, print_dev_f = _make_display_device_table(builder)
print(header_line)
for dev in builder._iter_devs():
flags = 'DEL' if dev in builder._remove_devs else ''
print_dev_f(dev, balance_per_dev[dev['id']], flags)
exit(EXIT_SUCCESS)
def all_bind_ports_for_node(self):
"""
Given an iterable of IP addresses identifying a storage backend server,
return a set of all bind ports defined in all rings for this storage
backend server.
The caller is responsible for not calling this method (which performs
at least a stat on all ring files) too frequently.
"""
# NOTE: we don't worry about disappearing rings here because you can't
# ever delete a storage policy.
for policy in POLICIES:
# NOTE: we must NOT use policy.load_ring to load the ring. Users
# of this utility function will not need the actual ring data, just
# the bind ports.
#
# This is duplicated with Ring.__init__ just a bit...
serialized_path = os.path.join(self.swift_dir,
policy.ring_name + '.ring.gz')
try:
new_mtime = os.path.getmtime(serialized_path)
except OSError:
continue
old_mtime = self.mtimes_by_ring_path.get(serialized_path)
if not old_mtime or old_mtime != new_mtime:
self.portsets_by_ring_path[serialized_path] = set(
dev['port']
for dev in RingData.load(serialized_path,
metadata_only=True).devs
if dev and dev['ip'] in self.my_ips)
self.mtimes_by_ring_path[serialized_path] = new_mtime
# No "break" here so that the above line will update the
# mtimes_by_ring_path entry for any ring that changes, not just
# the first one we notice.
# Return the requested set of ports from our (now-freshened) cache
return six.moves.reduce(set.union,
self.portsets_by_ring_path.values(), set())
def all_bind_ports_for_node(self):
"""
Given an iterable of IP addresses identifying a storage backend server,
return a set of all bind ports defined in all rings for this storage
backend server.
The caller is responsible for not calling this method (which performs
at least a stat on all ring files) too frequently.
"""
# NOTE: we don't worry about disappearing rings here because you can't
# ever delete a storage policy.
for policy in POLICIES:
# NOTE: we must NOT use policy.load_ring to load the ring. Users
# of this utility function will not need the actual ring data, just
# the bind ports.
#
# This is duplicated with Ring.__init__ just a bit...
serialized_path = os.path.join(self.swift_dir,
policy.ring_name + '.ring.gz')
try:
new_mtime = os.path.getmtime(serialized_path)
except OSError:
continue
old_mtime = self.mtimes_by_ring_path.get(serialized_path)
if not old_mtime or old_mtime != new_mtime:
self.portsets_by_ring_path[serialized_path] = set(
dev['port']
for dev in RingData.load(serialized_path,
metadata_only=True).devs
if dev and dev['ip'] in self.my_ips)
self.mtimes_by_ring_path[serialized_path] = new_mtime
# No "break" here so that the above line will update the
# mtimes_by_ring_path entry for any ring that changes, not just
# the first one we notice.
# Return the requested set of ports from our (now-freshened) cache
return six.moves.reduce(set.union, self.portsets_by_ring_path.values(),
set())
def default():
"""
swift-ring-builder <builder_file>
Shows information about the ring and the devices within.
Flags:
DEL - marked for removal and will be removed next rebalance.
"""
print('%s, build version %d' % (builder_file, builder.version))
regions = 0
zones = 0
balance = 0
dev_count = 0
if builder.devs:
regions = len(set(d['region'] for d in builder.devs
if d is not None))
zones = len(set((d['region'], d['zone']) for d in builder.devs
if d is not None))
dev_count = len([dev for dev in builder.devs
if dev is not None])
balance = builder.get_balance()
dispersion_trailer = '' if builder.dispersion is None else (
', %.02f dispersion' % (builder.dispersion))
print('%d partitions, %.6f replicas, %d regions, %d zones, '
'%d devices, %.02f balance%s' % (
builder.parts, builder.replicas, regions, zones, dev_count,
balance, dispersion_trailer))
print('The minimum number of hours before a partition can be '
'reassigned is %s (%s remaining)' % (
builder.min_part_hours,
timedelta(seconds=builder.min_part_seconds_left)))
print('The overload factor is %0.2f%% (%.6f)' % (
builder.overload * 100, builder.overload))
# compare ring file against builder file
if not exists(ring_file):
print('Ring file %s not found, '
'probably it hasn\'t been written yet' % ring_file)
else:
builder_dict = builder.get_ring().to_dict()
try:
ring_dict = RingData.load(ring_file).to_dict()
except Exception as exc:
print('Ring file %s is invalid: %r' % (ring_file, exc))
else:
if builder_dict == ring_dict:
print('Ring file %s is up-to-date' % ring_file)
else:
print('Ring file %s is obsolete' % ring_file)
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
print('Devices: id region zone ip address port '
'replication ip replication port name '
'weight partitions balance flags meta')
for dev in builder._iter_devs():
flags = 'DEL' if dev in builder._remove_devs else ''
print(' %5d %7d %5d %15s %5d %15s %17d %9s %6.02f '
'%10s %7.02f %5s %s' %
(dev['id'], dev['region'], dev['zone'], dev['ip'],
dev['port'], dev['replication_ip'],
dev['replication_port'], dev['device'], dev['weight'],
dev['parts'], balance_per_dev[dev['id']], flags,
dev['meta']))
exit(EXIT_SUCCESS)
def default():
"""
swift-ring-builder <builder_file>
Shows information about the ring and the devices within.
Flags:
DEL - marked for removal and will be removed next rebalance.
"""
print('%s, build version %d' % (builder_file, builder.version))
regions = 0
zones = 0
balance = 0
dev_count = 0
if builder.devs:
regions = len(
set(d['region'] for d in builder.devs if d is not None))
zones = len(
set((d['region'], d['zone']) for d in builder.devs
if d is not None))
dev_count = len([dev for dev in builder.devs if dev is not None])
balance = builder.get_balance()
dispersion_trailer = '' if builder.dispersion is None else (
', %.02f dispersion' % (builder.dispersion))
print('%d partitions, %.6f replicas, %d regions, %d zones, '
'%d devices, %.02f balance%s' %
(builder.parts, builder.replicas, regions, zones, dev_count,
balance, dispersion_trailer))
print('The minimum number of hours before a partition can be '
'reassigned is %s (%s remaining)' %
(builder.min_part_hours,
timedelta(seconds=builder.min_part_seconds_left)))
print('The overload factor is %0.2f%% (%.6f)' %
(builder.overload * 100, builder.overload))
# compare ring file against builder file
if not exists(ring_file):
print('Ring file %s not found, '
'probably it hasn\'t been written yet' % ring_file)
else:
builder_dict = builder.get_ring().to_dict()
try:
ring_dict = RingData.load(ring_file).to_dict()
except Exception as exc:
print('Ring file %s is invalid: %r' % (ring_file, exc))
else:
if builder_dict == ring_dict:
print('Ring file %s is up-to-date' % ring_file)
else:
print('Ring file %s is obsolete' % ring_file)
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
print('Devices: id region zone ip address port '
'replication ip replication port name '
'weight partitions balance flags meta')
for dev in builder._iter_devs():
flags = 'DEL' if dev in builder._remove_devs else ''
print(' %5d %7d %5d %15s %5d %15s %17d %9s %6.02f '
'%10s %7.02f %5s %s' %
(dev['id'], dev['region'], dev['zone'], dev['ip'],
dev['port'], dev['replication_ip'],
dev['replication_port'], dev['device'], dev['weight'],
dev['parts'], balance_per_dev[dev['id']], flags,
dev['meta']))
exit(EXIT_SUCCESS)
def default():
"""
swift-ring-builder <builder_file>
Shows information about the ring and the devices within.
Flags:
DEL - marked for removal and will be removed next rebalance.
"""
print("%s, build version %d" % (builder_file, builder.version))
regions = 0
zones = 0
balance = 0
dev_count = 0
if builder.devs:
regions = len(set(d["region"] for d in builder.devs if d is not None))
zones = len(set((d["region"], d["zone"]) for d in builder.devs if d is not None))
dev_count = len([dev for dev in builder.devs if dev is not None])
balance = builder.get_balance()
dispersion_trailer = "" if builder.dispersion is None else (", %.02f dispersion" % (builder.dispersion))
print(
"%d partitions, %.6f replicas, %d regions, %d zones, "
"%d devices, %.02f balance%s"
% (builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer)
)
print(
"The minimum number of hours before a partition can be "
"reassigned is %s (%s remaining)"
% (builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left))
)
print("The overload factor is %0.2f%% (%.6f)" % (builder.overload * 100, builder.overload))
# compare ring file against builder file
if not exists(ring_file):
print("Ring file %s not found, " "probably it hasn't been written yet" % ring_file)
else:
builder_dict = builder.get_ring().to_dict()
try:
ring_dict = RingData.load(ring_file).to_dict()
except Exception as exc:
print("Ring file %s is invalid: %r" % (ring_file, exc))
else:
if builder_dict == ring_dict:
print("Ring file %s is up-to-date" % ring_file)
else:
print("Ring file %s is obsolete" % ring_file)
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
print(
"Devices: id region zone ip address port "
"replication ip replication port name "
"weight partitions balance flags meta"
)
for dev in builder._iter_devs():
flags = "DEL" if dev in builder._remove_devs else ""
print(
" %5d %7d %5d %15s %5d %15s %17d %9s %6.02f "
"%10s %7.02f %5s %s"
% (
dev["id"],
dev["region"],
dev["zone"],
dev["ip"],
dev["port"],
dev["replication_ip"],
dev["replication_port"],
dev["device"],
dev["weight"],
dev["parts"],
balance_per_dev[dev["id"]],
flags,
dev["meta"],
)
)
exit(EXIT_SUCCESS)
from swift.common.ring import RingData, RingBuilder
from array import array
import math
import pickle
"""
https://docs.openstack.org/swift/pike/admin/objectstorage-troubleshoot.html
Using existing swift tools, there is no way to recover a builder file from a ring.gz file. However,
if you have a knowledge of Python, it is possible to construct a builder file that is pretty close to the one you have lost.
"""
ring = RingData.load('/home/shahbazi/Desktop/rings/account.ring.gz')
partitions = len(ring._replica2part2dev_id[0])
replicas = len(ring._replica2part2dev_id)
builder = RingBuilder(int(math.log(partitions, 2)), replicas, 1)
builder.devs = ring.devs
builder._replica2part2dev = ring._replica2part2dev_id
builder._last_part_moves_epoch = 0
builder._last_part_moves = array('B', (0 for _ in range(partitions)))
builder.change_min_part_hours(24)
# builder._set_parts_wanted()
def main():
usage = "usage: %prog [options] arg"
parser = OptionParser(usage)
group = OptionGroup(parser, "Moving Map Building")
group.add_option("-o",
"--old_ring",
dest="old_ring_path",
help="The path to the old ring file.")
group.add_option("-n",
"--new_ring",
dest="new_ring_path",
help="The path to the new ring file.")
group.add_option("-t",
"--test",
dest="test",
default="False",
help="Controls whether the execution will only printout "
"the commands to standard output, "
"without explicitly run them. To set (True) "
"to unset (False). The default value is False. "
"Used for manual testing only.")
group.add_option("-r",
"--run",
dest="run",
default="False",
help="Controls the data movement script execution. "
" Default is False")
group.add_option("-f",
"--file",
dest="dump_file",
help="The path to the data moving map dump file.")
group.add_option("-c",
"--concurrency",
dest="concurrency",
type="int",
default=1,
help="The concurrency level, the default is 1.")
group.add_option("-m",
"--mover_tmp_dir",
dest="mover_tmp_dir",
default='data_mover',
help="The name of temporal directory that would "
"be used for data migration")
parser.add_option_group(group)
(options, args) = parser.parse_args()
validate_moving_map_options(options, args)
if options.dump_file:
dump_file = options.dump_file
else:
dump_file = DEFAULT_DUMP_FILE
if bool(options.run):
conf = {}
conf['data_moving_map_dump'] = dump_file
conf['concurrency'] = options.concurrency
conf['test'] = options.test
conf['mover_tmp_dir'] = options.mover_tmp_dir
objMover = ObjectMover(conf=conf)
objMover.run_once()
else:
old_ring_data = RingData.load(options.old_ring_path)
new_ring_data = RingData.load(options.new_ring_path)
if bool(options.test):
print_moving_map(old_ring_data, new_ring_data, 10)
dump_moving_map(old_ring_data, new_ring_data, dump_file)
if bool(options.test):
load_moving_map(dump_file, True)
def main():
usage = "usage: %prog [options] arg"
parser = OptionParser(usage)
group = OptionGroup(parser, "Moving Map Building")
group.add_option("-o", "--old_ring", dest="old_ring_path",
help="The path to the old ring file.")
group.add_option("-n", "--new_ring", dest="new_ring_path",
help="The path to the new ring file.")
group.add_option("-t", "--test", dest="test", default="False",
help="Controls whether the execution will only printout "
"the commands to standard output, "
"without explicitly run them. To set (True) "
"to unset (False). The default value is False. "
"Used for manual testing only.")
group.add_option("-r", "--run", dest="run", default="False",
help="Controls the data movement script execution. "
" Default is False")
group.add_option("-f", "--file", dest="dump_file",
help="The path to the data moving map dump file.")
group.add_option("-c", "--concurrency", dest="concurrency", type="int",
default=1,
help="The concurrency level, the default is 1.")
group.add_option("-m", "--mover_tmp_dir", dest="mover_tmp_dir",
default='data_mover',
help="The name of temporal directory that would "
"be used for data migration")
parser.add_option_group(group)
(options, args) = parser.parse_args()
validate_moving_map_options(options, args)
if options.dump_file:
dump_file = options.dump_file
else:
dump_file = DEFAULT_DUMP_FILE
if bool(options.run):
conf = {}
conf['data_moving_map_dump'] = dump_file
conf['concurrency'] = options.concurrency
conf['test'] = options.test
conf['mover_tmp_dir'] = options.mover_tmp_dir
objMover = ObjectMover(conf=conf)
objMover.run_once()
else:
old_ring_data = RingData.load(options.old_ring_path)
new_ring_data = RingData.load(options.new_ring_path)
if bool(options.test):
print_moving_map(old_ring_data, new_ring_data, 10)
dump_moving_map(old_ring_data, new_ring_data, dump_file)
if bool(options.test):
load_moving_map(dump_file, True)
