def default(): """ swift-ring-builder <builder_file> Shows information about the ring and the devices within. Output includes a table that describes the report parameters (id, region, port, flags, etc). flags: possible values are 'DEL' and '' DEL - indicates that the device is marked for removal from ring and will be removed in next rebalance. """ print('%s, build version %d' % (builder_file, builder.version)) regions = 0 zones = 0 balance = 0 dev_count = 0 if builder.devs: regions = len(set(d['region'] for d in builder.devs if d is not None)) zones = len(set((d['region'], d['zone']) for d in builder.devs if d is not None)) dev_count = len([dev for dev in builder.devs if dev is not None]) balance = builder.get_balance() dispersion_trailer = '' if builder.dispersion is None else ( ', %.02f dispersion' % (builder.dispersion)) print('%d partitions, %.6f replicas, %d regions, %d zones, ' '%d devices, %.02f balance%s' % ( builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer)) print('The minimum number of hours before a partition can be ' 'reassigned is %s (%s remaining)' % ( builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left))) print('The overload factor is %0.2f%% (%.6f)' % ( builder.overload * 100, builder.overload)) # compare ring file against builder file if not exists(ring_file): print('Ring file %s not found, ' 'probably it hasn\'t been written yet' % ring_file) else: builder_dict = builder.get_ring().to_dict() try: ring_dict = RingData.load(ring_file).to_dict() except Exception as exc: print('Ring file %s is invalid: %r' % (ring_file, exc)) else: if builder_dict == ring_dict: print('Ring file %s is up-to-date' % ring_file) else: print('Ring file %s is obsolete' % ring_file) if builder.devs: balance_per_dev = builder._build_balance_per_dev() header_line, print_dev_f = _make_display_device_table(builder) print(header_line) for dev in builder._iter_devs(): flags = 'DEL' if dev in builder._remove_devs else '' print_dev_f(dev, balance_per_dev[dev['id']], flags) exit(EXIT_SUCCESS)
def all_bind_ports_for_node(self): """ Given an iterable of IP addresses identifying a storage backend server, return a set of all bind ports defined in all rings for this storage backend server. The caller is responsible for not calling this method (which performs at least a stat on all ring files) too frequently. """ # NOTE: we don't worry about disappearing rings here because you can't # ever delete a storage policy. for policy in POLICIES: # NOTE: we must NOT use policy.load_ring to load the ring. Users # of this utility function will not need the actual ring data, just # the bind ports. # # This is duplicated with Ring.__init__ just a bit... serialized_path = os.path.join(self.swift_dir, policy.ring_name + '.ring.gz') try: new_mtime = os.path.getmtime(serialized_path) except OSError: continue old_mtime = self.mtimes_by_ring_path.get(serialized_path) if not old_mtime or old_mtime != new_mtime: self.portsets_by_ring_path[serialized_path] = set( dev['port'] for dev in RingData.load(serialized_path, metadata_only=True).devs if dev and dev['ip'] in self.my_ips) self.mtimes_by_ring_path[serialized_path] = new_mtime # No "break" here so that the above line will update the # mtimes_by_ring_path entry for any ring that changes, not just # the first one we notice. # Return the requested set of ports from our (now-freshened) cache return six.moves.reduce(set.union, self.portsets_by_ring_path.values(), set())
def default(): """ swift-ring-builder <builder_file> Shows information about the ring and the devices within. Flags: DEL - marked for removal and will be removed next rebalance. """ print('%s, build version %d' % (builder_file, builder.version)) regions = 0 zones = 0 balance = 0 dev_count = 0 if builder.devs: regions = len(set(d['region'] for d in builder.devs if d is not None)) zones = len(set((d['region'], d['zone']) for d in builder.devs if d is not None)) dev_count = len([dev for dev in builder.devs if dev is not None]) balance = builder.get_balance() dispersion_trailer = '' if builder.dispersion is None else ( ', %.02f dispersion' % (builder.dispersion)) print('%d partitions, %.6f replicas, %d regions, %d zones, ' '%d devices, %.02f balance%s' % ( builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer)) print('The minimum number of hours before a partition can be ' 'reassigned is %s (%s remaining)' % ( builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left))) print('The overload factor is %0.2f%% (%.6f)' % ( builder.overload * 100, builder.overload)) # compare ring file against builder file if not exists(ring_file): print('Ring file %s not found, ' 'probably it hasn\'t been written yet' % ring_file) else: builder_dict = builder.get_ring().to_dict() try: ring_dict = RingData.load(ring_file).to_dict() except Exception as exc: print('Ring file %s is invalid: %r' % (ring_file, exc)) else: if builder_dict == ring_dict: print('Ring file %s is up-to-date' % ring_file) else: print('Ring file %s is obsolete' % ring_file) if builder.devs: balance_per_dev = builder._build_balance_per_dev() print('Devices: id region zone ip address port ' 'replication ip replication port name ' 'weight partitions balance flags meta') for dev in builder._iter_devs(): flags = 'DEL' if dev in builder._remove_devs else '' print(' %5d %7d %5d %15s %5d %15s %17d %9s %6.02f ' '%10s %7.02f %5s %s' % (dev['id'], dev['region'], dev['zone'], dev['ip'], dev['port'], dev['replication_ip'], dev['replication_port'], dev['device'], dev['weight'], dev['parts'], balance_per_dev[dev['id']], flags, dev['meta'])) exit(EXIT_SUCCESS)
def default(): """ swift-ring-builder <builder_file> Shows information about the ring and the devices within. Flags: DEL - marked for removal and will be removed next rebalance. """ print('%s, build version %d' % (builder_file, builder.version)) regions = 0 zones = 0 balance = 0 dev_count = 0 if builder.devs: regions = len( set(d['region'] for d in builder.devs if d is not None)) zones = len( set((d['region'], d['zone']) for d in builder.devs if d is not None)) dev_count = len([dev for dev in builder.devs if dev is not None]) balance = builder.get_balance() dispersion_trailer = '' if builder.dispersion is None else ( ', %.02f dispersion' % (builder.dispersion)) print('%d partitions, %.6f replicas, %d regions, %d zones, ' '%d devices, %.02f balance%s' % (builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer)) print('The minimum number of hours before a partition can be ' 'reassigned is %s (%s remaining)' % (builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left))) print('The overload factor is %0.2f%% (%.6f)' % (builder.overload * 100, builder.overload)) # compare ring file against builder file if not exists(ring_file): print('Ring file %s not found, ' 'probably it hasn\'t been written yet' % ring_file) else: builder_dict = builder.get_ring().to_dict() try: ring_dict = RingData.load(ring_file).to_dict() except Exception as exc: print('Ring file %s is invalid: %r' % (ring_file, exc)) else: if builder_dict == ring_dict: print('Ring file %s is up-to-date' % ring_file) else: print('Ring file %s is obsolete' % ring_file) if builder.devs: balance_per_dev = builder._build_balance_per_dev() print('Devices: id region zone ip address port ' 'replication ip replication port name ' 'weight partitions balance flags meta') for dev in builder._iter_devs(): flags = 'DEL' if dev in builder._remove_devs else '' print(' %5d %7d %5d %15s %5d %15s %17d %9s %6.02f ' '%10s %7.02f %5s %s' % (dev['id'], dev['region'], dev['zone'], dev['ip'], dev['port'], dev['replication_ip'], dev['replication_port'], dev['device'], dev['weight'], dev['parts'], balance_per_dev[dev['id']], flags, dev['meta'])) exit(EXIT_SUCCESS)
def default(): """ swift-ring-builder <builder_file> Shows information about the ring and the devices within. Flags: DEL - marked for removal and will be removed next rebalance. """ print("%s, build version %d" % (builder_file, builder.version)) regions = 0 zones = 0 balance = 0 dev_count = 0 if builder.devs: regions = len(set(d["region"] for d in builder.devs if d is not None)) zones = len(set((d["region"], d["zone"]) for d in builder.devs if d is not None)) dev_count = len([dev for dev in builder.devs if dev is not None]) balance = builder.get_balance() dispersion_trailer = "" if builder.dispersion is None else (", %.02f dispersion" % (builder.dispersion)) print( "%d partitions, %.6f replicas, %d regions, %d zones, " "%d devices, %.02f balance%s" % (builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer) ) print( "The minimum number of hours before a partition can be " "reassigned is %s (%s remaining)" % (builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left)) ) print("The overload factor is %0.2f%% (%.6f)" % (builder.overload * 100, builder.overload)) # compare ring file against builder file if not exists(ring_file): print("Ring file %s not found, " "probably it hasn't been written yet" % ring_file) else: builder_dict = builder.get_ring().to_dict() try: ring_dict = RingData.load(ring_file).to_dict() except Exception as exc: print("Ring file %s is invalid: %r" % (ring_file, exc)) else: if builder_dict == ring_dict: print("Ring file %s is up-to-date" % ring_file) else: print("Ring file %s is obsolete" % ring_file) if builder.devs: balance_per_dev = builder._build_balance_per_dev() print( "Devices: id region zone ip address port " "replication ip replication port name " "weight partitions balance flags meta" ) for dev in builder._iter_devs(): flags = "DEL" if dev in builder._remove_devs else "" print( " %5d %7d %5d %15s %5d %15s %17d %9s %6.02f " "%10s %7.02f %5s %s" % ( dev["id"], dev["region"], dev["zone"], dev["ip"], dev["port"], dev["replication_ip"], dev["replication_port"], dev["device"], dev["weight"], dev["parts"], balance_per_dev[dev["id"]], flags, dev["meta"], ) ) exit(EXIT_SUCCESS)
from swift.common.ring import RingData, RingBuilder from array import array import math import pickle """ https://docs.openstack.org/swift/pike/admin/objectstorage-troubleshoot.html Using existing swift tools, there is no way to recover a builder file from a ring.gz file. However, if you have a knowledge of Python, it is possible to construct a builder file that is pretty close to the one you have lost. """ ring = RingData.load('/home/shahbazi/Desktop/rings/account.ring.gz') partitions = len(ring._replica2part2dev_id[0]) replicas = len(ring._replica2part2dev_id) builder = RingBuilder(int(math.log(partitions, 2)), replicas, 1) builder.devs = ring.devs builder._replica2part2dev = ring._replica2part2dev_id builder._last_part_moves_epoch = 0 builder._last_part_moves = array('B', (0 for _ in range(partitions))) builder.change_min_part_hours(24) # builder._set_parts_wanted()
def main(): usage = "usage: %prog [options] arg" parser = OptionParser(usage) group = OptionGroup(parser, "Moving Map Building") group.add_option("-o", "--old_ring", dest="old_ring_path", help="The path to the old ring file.") group.add_option("-n", "--new_ring", dest="new_ring_path", help="The path to the new ring file.") group.add_option("-t", "--test", dest="test", default="False", help="Controls whether the execution will only printout " "the commands to standard output, " "without explicitly run them. To set (True) " "to unset (False). The default value is False. " "Used for manual testing only.") group.add_option("-r", "--run", dest="run", default="False", help="Controls the data movement script execution. " " Default is False") group.add_option("-f", "--file", dest="dump_file", help="The path to the data moving map dump file.") group.add_option("-c", "--concurrency", dest="concurrency", type="int", default=1, help="The concurrency level, the default is 1.") group.add_option("-m", "--mover_tmp_dir", dest="mover_tmp_dir", default='data_mover', help="The name of temporal directory that would " "be used for data migration") parser.add_option_group(group) (options, args) = parser.parse_args() validate_moving_map_options(options, args) if options.dump_file: dump_file = options.dump_file else: dump_file = DEFAULT_DUMP_FILE if bool(options.run): conf = {} conf['data_moving_map_dump'] = dump_file conf['concurrency'] = options.concurrency conf['test'] = options.test conf['mover_tmp_dir'] = options.mover_tmp_dir objMover = ObjectMover(conf=conf) objMover.run_once() else: old_ring_data = RingData.load(options.old_ring_path) new_ring_data = RingData.load(options.new_ring_path) if bool(options.test): print_moving_map(old_ring_data, new_ring_data, 10) dump_moving_map(old_ring_data, new_ring_data, dump_file) if bool(options.test): load_moving_map(dump_file, True)