def default(): """ swift-ring-builder <builder_file> Shows information about the ring and the devices within. Output includes a table that describes the report parameters (id, region, port, flags, etc). flags: possible values are 'DEL' and '' DEL - indicates that the device is marked for removal from ring and will be removed in next rebalance. """ print('%s, build version %d' % (builder_file, builder.version)) regions = 0 zones = 0 balance = 0 dev_count = 0 if builder.devs: regions = len(set(d['region'] for d in builder.devs if d is not None)) zones = len(set((d['region'], d['zone']) for d in builder.devs if d is not None)) dev_count = len([dev for dev in builder.devs if dev is not None]) balance = builder.get_balance() dispersion_trailer = '' if builder.dispersion is None else ( ', %.02f dispersion' % (builder.dispersion)) print('%d partitions, %.6f replicas, %d regions, %d zones, ' '%d devices, %.02f balance%s' % ( builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer)) print('The minimum number of hours before a partition can be ' 'reassigned is %s (%s remaining)' % ( builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left))) print('The overload factor is %0.2f%% (%.6f)' % ( builder.overload * 100, builder.overload)) # compare ring file against builder file if not exists(ring_file): print('Ring file %s not found, ' 'probably it hasn\'t been written yet' % ring_file) else: builder_dict = builder.get_ring().to_dict() try: ring_dict = RingData.load(ring_file).to_dict() except Exception as exc: print('Ring file %s is invalid: %r' % (ring_file, exc)) else: if builder_dict == ring_dict: print('Ring file %s is up-to-date' % ring_file) else: print('Ring file %s is obsolete' % ring_file) if builder.devs: balance_per_dev = builder._build_balance_per_dev() header_line, print_dev_f = _make_display_device_table(builder) print(header_line) for dev in builder._iter_devs(): flags = 'DEL' if dev in builder._remove_devs else '' print_dev_f(dev, balance_per_dev[dev['id']], flags) exit(EXIT_SUCCESS)
def setUp(self): utils.HASH_PATH_SUFFIX = b'endcap' utils.HASH_PATH_PREFIX = b'' self.testdir = mkdtemp() ring_file = os.path.join(self.testdir, 'container.ring.gz') with closing(GzipFile(ring_file, 'wb')) as f: pickle.dump( RingData([[0, 1, 2, 0, 1, 2], [1, 2, 0, 1, 2, 0], [2, 3, 1, 2, 3, 1]], [{ 'id': 0, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', 'zone': 0 }, { 'id': 1, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', 'zone': 2 }, { 'id': 2, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', 'zone': 4 }, { 'id': 3, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', 'zone': 6 }], 30), f) self.devices_dir = os.path.join(self.testdir, 'devices') os.mkdir(self.devices_dir) self.sda1 = os.path.join(self.devices_dir, 'sda1') os.mkdir(self.sda1) for policy in POLICIES: os.mkdir(os.path.join(self.sda1, get_tmp_dir(policy))) self.logger = debug_logger() self.ts_iter = make_timestamp_iter()
def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = '' self.testdir = os.path.join(os.path.dirname(__file__), 'object_updater') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) ring_file = os.path.join(self.testdir, 'container.ring.gz') with closing(GzipFile(ring_file, 'wb')) as f: pickle.dump( RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', 'zone': 0}, {'id': 1, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', 'zone': 2}], 30), f) self.devices_dir = os.path.join(self.testdir, 'devices') os.mkdir(self.devices_dir) self.sda1 = os.path.join(self.devices_dir, 'sda1') os.mkdir(self.sda1) os.mkdir(os.path.join(self.sda1, 'tmp'))
def all_bind_ports_for_node(self): """ Given an iterable of IP addresses identifying a storage backend server, return a set of all bind ports defined in all rings for this storage backend server. The caller is responsible for not calling this method (which performs at least a stat on all ring files) too frequently. """ # NOTE: we don't worry about disappearing rings here because you can't # ever delete a storage policy. for policy in POLICIES: # NOTE: we must NOT use policy.load_ring to load the ring. Users # of this utility function will not need the actual ring data, just # the bind ports. # # This is duplicated with Ring.__init__ just a bit... serialized_path = os.path.join(self.swift_dir, policy.ring_name + '.ring.gz') try: new_mtime = os.path.getmtime(serialized_path) except OSError: continue old_mtime = self.mtimes_by_ring_path.get(serialized_path) if not old_mtime or old_mtime != new_mtime: self.portsets_by_ring_path[serialized_path] = set( dev['port'] for dev in RingData.load(serialized_path, metadata_only=True).devs if dev and dev['ip'] in self.my_ips) self.mtimes_by_ring_path[serialized_path] = new_mtime # No "break" here so that the above line will update the # mtimes_by_ring_path entry for any ring that changes, not just # the first one we notice. # Return the requested set of ports from our (now-freshened) cache return six.moves.reduce(set.union, self.portsets_by_ring_path.values(), set())
def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' self.testdir = os.path.join(mkdtemp(), 'tmp_test_container_updater') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) pickle.dump( RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{ 'id': 0, 'ip': '127.0.0.1', 'port': 12345, 'device': 'sda1', 'zone': 0 }, { 'id': 1, 'ip': '127.0.0.1', 'port': 12345, 'device': 'sda1', 'zone': 2 }], 30), GzipFile(os.path.join(self.testdir, 'account.ring.gz'), 'wb')) self.devices_dir = os.path.join(self.testdir, 'devices') os.mkdir(self.devices_dir) self.sda1 = os.path.join(self.devices_dir, 'sda1') os.mkdir(self.sda1)
def default(): """ swift-ring-builder <builder_file> Shows information about the ring and the devices within. Flags: DEL - marked for removal and will be removed next rebalance. """ print('%s, build version %d' % (builder_file, builder.version)) regions = 0 zones = 0 balance = 0 dev_count = 0 if builder.devs: regions = len(set(d['region'] for d in builder.devs if d is not None)) zones = len(set((d['region'], d['zone']) for d in builder.devs if d is not None)) dev_count = len([dev for dev in builder.devs if dev is not None]) balance = builder.get_balance() dispersion_trailer = '' if builder.dispersion is None else ( ', %.02f dispersion' % (builder.dispersion)) print('%d partitions, %.6f replicas, %d regions, %d zones, ' '%d devices, %.02f balance%s' % ( builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer)) print('The minimum number of hours before a partition can be ' 'reassigned is %s (%s remaining)' % ( builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left))) print('The overload factor is %0.2f%% (%.6f)' % ( builder.overload * 100, builder.overload)) # compare ring file against builder file if not exists(ring_file): print('Ring file %s not found, ' 'probably it hasn\'t been written yet' % ring_file) else: builder_dict = builder.get_ring().to_dict() try: ring_dict = RingData.load(ring_file).to_dict() except Exception as exc: print('Ring file %s is invalid: %r' % (ring_file, exc)) else: if builder_dict == ring_dict: print('Ring file %s is up-to-date' % ring_file) else: print('Ring file %s is obsolete' % ring_file) if builder.devs: balance_per_dev = builder._build_balance_per_dev() print('Devices: id region zone ip address port ' 'replication ip replication port name ' 'weight partitions balance flags meta') for dev in builder._iter_devs(): flags = 'DEL' if dev in builder._remove_devs else '' print(' %5d %7d %5d %15s %5d %15s %17d %9s %6.02f ' '%10s %7.02f %5s %s' % (dev['id'], dev['region'], dev['zone'], dev['ip'], dev['port'], dev['replication_ip'], dev['replication_port'], dev['device'], dev['weight'], dev['parts'], balance_per_dev[dev['id']], flags, dev['meta'])) exit(EXIT_SUCCESS)
def setup_servers(the_object_server=object_server, extra_conf=None): """ Setup proxy, account, container and object servers using a set of fake rings and policies. :param the_object_server: The object server module to use (optional, defaults to swift.obj.server) :param extra_conf: A dict of config options that will update the basic config passed to all server instances. :returns: A dict containing the following entries: orig_POLICIES: the value of storage_policy.POLICIES prior to it being patched with fake policies orig_SysLogHandler: the value of utils.SysLogHandler prior to it being patched testdir: root directory used for test files test_POLICIES: a StoragePolicyCollection of fake policies test_servers: a tuple of test server instances test_sockets: a tuple of sockets used by test servers test_coros: a tuple of greenthreads in which test servers are running """ context = { "orig_POLICIES": storage_policy._POLICIES, "orig_SysLogHandler": utils.SysLogHandler} utils.HASH_PATH_SUFFIX = b'endcap' utils.SysLogHandler = mock.MagicMock() # Since we're starting up a lot here, we're going to test more than # just chunked puts; we're also going to test parts of # proxy_server.Application we couldn't get to easily otherwise. context["testdir"] = _testdir = \ os.path.join(mkdtemp(), 'tmp_test_proxy_server_chunked') mkdirs(_testdir) rmtree(_testdir) for drive in ('sda1', 'sdb1', 'sdc1', 'sdd1', 'sde1', 'sdf1', 'sdg1', 'sdh1', 'sdi1', 'sdj1', 'sdk1', 'sdl1'): mkdirs(os.path.join(_testdir, drive, 'tmp')) conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': 'content-encoding, x-object-manifest, content-disposition, foo', 'allow_versions': 't'} if extra_conf: conf.update(extra_conf) prolis = listen_zero() acc1lis = listen_zero() acc2lis = listen_zero() con1lis = listen_zero() con2lis = listen_zero() obj1lis = listen_zero() obj2lis = listen_zero() obj3lis = listen_zero() obj4lis = listen_zero() obj5lis = listen_zero() obj6lis = listen_zero() objsocks = [obj1lis, obj2lis, obj3lis, obj4lis, obj5lis, obj6lis] context["test_sockets"] = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, obj3lis, obj4lis, obj5lis, obj6lis) account_ring_path = os.path.join(_testdir, 'account.ring.gz') account_devs = [ {'port': acc1lis.getsockname()[1]}, {'port': acc2lis.getsockname()[1]}, ] write_fake_ring(account_ring_path, *account_devs) container_ring_path = os.path.join(_testdir, 'container.ring.gz') container_devs = [ {'port': con1lis.getsockname()[1]}, {'port': con2lis.getsockname()[1]}, ] write_fake_ring(container_ring_path, *container_devs) storage_policy._POLICIES = storage_policy.StoragePolicyCollection([ StoragePolicy(0, 'zero', True), StoragePolicy(1, 'one', False), StoragePolicy(2, 'two', False), ECStoragePolicy(3, 'ec', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, ec_segment_size=4096), ECStoragePolicy(4, 'ec-dup', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, ec_segment_size=4096, ec_duplication_factor=2)]) obj_rings = { 0: ('sda1', 'sdb1'), 1: ('sdc1', 'sdd1'), 2: ('sde1', 'sdf1'), # sdg1, sdh1, sdi1 taken by policy 3 (see below) } for policy_index, devices in obj_rings.items(): policy = storage_policy.POLICIES[policy_index] obj_ring_path = os.path.join(_testdir, policy.ring_name + '.ring.gz') obj_devs = [ {'port': objsock.getsockname()[1], 'device': dev} for objsock, dev in zip(objsocks, devices)] write_fake_ring(obj_ring_path, *obj_devs) # write_fake_ring can't handle a 3-element ring, and the EC policy needs # at least 6 devs to work with (ec_k=2, ec_m=1, duplication_factor=2), # so we do it manually devs = [{'id': 0, 'zone': 0, 'device': 'sdg1', 'ip': '127.0.0.1', 'port': obj1lis.getsockname()[1]}, {'id': 1, 'zone': 0, 'device': 'sdh1', 'ip': '127.0.0.1', 'port': obj2lis.getsockname()[1]}, {'id': 2, 'zone': 0, 'device': 'sdi1', 'ip': '127.0.0.1', 'port': obj3lis.getsockname()[1]}, {'id': 3, 'zone': 0, 'device': 'sdj1', 'ip': '127.0.0.1', 'port': obj4lis.getsockname()[1]}, {'id': 4, 'zone': 0, 'device': 'sdk1', 'ip': '127.0.0.1', 'port': obj5lis.getsockname()[1]}, {'id': 5, 'zone': 0, 'device': 'sdl1', 'ip': '127.0.0.1', 'port': obj6lis.getsockname()[1]}] pol3_replica2part2dev_id = [[0, 1, 2, 0], [1, 2, 0, 1], [2, 0, 1, 2]] pol4_replica2part2dev_id = [[0, 1, 2, 3], [1, 2, 3, 4], [2, 3, 4, 5], [3, 4, 5, 0], [4, 5, 0, 1], [5, 0, 1, 2]] obj3_ring_path = os.path.join( _testdir, storage_policy.POLICIES[3].ring_name + '.ring.gz') part_shift = 30 with closing(GzipFile(obj3_ring_path, 'wb')) as fh: pickle.dump(RingData(pol3_replica2part2dev_id, devs, part_shift), fh) obj4_ring_path = os.path.join( _testdir, storage_policy.POLICIES[4].ring_name + '.ring.gz') part_shift = 30 with closing(GzipFile(obj4_ring_path, 'wb')) as fh: pickle.dump(RingData(pol4_replica2part2dev_id, devs, part_shift), fh) prosrv = proxy_server.Application(conf, logger=debug_logger('proxy')) for policy in storage_policy.POLICIES: # make sure all the rings are loaded prosrv.get_object_ring(policy.idx) # don't lose this one! context["test_POLICIES"] = storage_policy._POLICIES acc1srv = account_server.AccountController( conf, logger=debug_logger('acct1')) acc2srv = account_server.AccountController( conf, logger=debug_logger('acct2')) con1srv = container_server.ContainerController( conf, logger=debug_logger('cont1')) con2srv = container_server.ContainerController( conf, logger=debug_logger('cont2')) obj1srv = the_object_server.ObjectController( conf, logger=debug_logger('obj1')) obj2srv = the_object_server.ObjectController( conf, logger=debug_logger('obj2')) obj3srv = the_object_server.ObjectController( conf, logger=debug_logger('obj3')) obj4srv = the_object_server.ObjectController( conf, logger=debug_logger('obj4')) obj5srv = the_object_server.ObjectController( conf, logger=debug_logger('obj5')) obj6srv = the_object_server.ObjectController( conf, logger=debug_logger('obj6')) context["test_servers"] = \ (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv, obj3srv, obj4srv, obj5srv, obj6srv) nl = NullLogger() logging_prosv = proxy_logging.ProxyLoggingMiddleware( listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger) prospa = spawn(wsgi.server, prolis, logging_prosv, nl) acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl) acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl) con1spa = spawn(wsgi.server, con1lis, con1srv, nl) con2spa = spawn(wsgi.server, con2lis, con2srv, nl) obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl) obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl) obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl) obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl) obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl) obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl) context["test_coros"] = \ (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa, obj4spa, obj5spa, obj6spa) # Create account ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a') for node in nodes: conn = swift.proxy.controllers.obj.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a', {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() assert(resp.status == 201) # Create another account # used for account-to-account tests ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a1') for node in nodes: conn = swift.proxy.controllers.obj.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a1', {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() assert(resp.status == 201) # Create containers, 1 per test policy sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( exp, headers[:len(exp)]) # Create container in other account # used for account-to-account tests sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a1/c1 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( exp, headers[:len(exp)]) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write( 'PUT /v1/a/c1 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: one\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, \ "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write( 'PUT /v1/a/c2 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: two\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, \ "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) return context
def default(): """ swift-ring-builder <builder_file> Shows information about the ring and the devices within. Flags: DEL - marked for removal and will be removed next rebalance. """ print('%s, build version %d' % (builder_file, builder.version)) regions = 0 zones = 0 balance = 0 dev_count = 0 if builder.devs: regions = len( set(d['region'] for d in builder.devs if d is not None)) zones = len( set((d['region'], d['zone']) for d in builder.devs if d is not None)) dev_count = len([dev for dev in builder.devs if dev is not None]) balance = builder.get_balance() dispersion_trailer = '' if builder.dispersion is None else ( ', %.02f dispersion' % (builder.dispersion)) print('%d partitions, %.6f replicas, %d regions, %d zones, ' '%d devices, %.02f balance%s' % (builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer)) print('The minimum number of hours before a partition can be ' 'reassigned is %s (%s remaining)' % (builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left))) print('The overload factor is %0.2f%% (%.6f)' % (builder.overload * 100, builder.overload)) # compare ring file against builder file if not exists(ring_file): print('Ring file %s not found, ' 'probably it hasn\'t been written yet' % ring_file) else: builder_dict = builder.get_ring().to_dict() try: ring_dict = RingData.load(ring_file).to_dict() except Exception as exc: print('Ring file %s is invalid: %r' % (ring_file, exc)) else: if builder_dict == ring_dict: print('Ring file %s is up-to-date' % ring_file) else: print('Ring file %s is obsolete' % ring_file) if builder.devs: balance_per_dev = builder._build_balance_per_dev() print('Devices: id region zone ip address port ' 'replication ip replication port name ' 'weight partitions balance flags meta') for dev in builder._iter_devs(): flags = 'DEL' if dev in builder._remove_devs else '' print(' %5d %7d %5d %15s %5d %15s %17d %9s %6.02f ' '%10s %7.02f %5s %s' % (dev['id'], dev['region'], dev['zone'], dev['ip'], dev['port'], dev['replication_ip'], dev['replication_port'], dev['device'], dev['weight'], dev['parts'], balance_per_dev[dev['id']], flags, dev['meta'])) exit(EXIT_SUCCESS)
def default(): """ swift-ring-builder <builder_file> Shows information about the ring and the devices within. Flags: DEL - marked for removal and will be removed next rebalance. """ print("%s, build version %d" % (builder_file, builder.version)) regions = 0 zones = 0 balance = 0 dev_count = 0 if builder.devs: regions = len(set(d["region"] for d in builder.devs if d is not None)) zones = len(set((d["region"], d["zone"]) for d in builder.devs if d is not None)) dev_count = len([dev for dev in builder.devs if dev is not None]) balance = builder.get_balance() dispersion_trailer = "" if builder.dispersion is None else (", %.02f dispersion" % (builder.dispersion)) print( "%d partitions, %.6f replicas, %d regions, %d zones, " "%d devices, %.02f balance%s" % (builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer) ) print( "The minimum number of hours before a partition can be " "reassigned is %s (%s remaining)" % (builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left)) ) print("The overload factor is %0.2f%% (%.6f)" % (builder.overload * 100, builder.overload)) # compare ring file against builder file if not exists(ring_file): print("Ring file %s not found, " "probably it hasn't been written yet" % ring_file) else: builder_dict = builder.get_ring().to_dict() try: ring_dict = RingData.load(ring_file).to_dict() except Exception as exc: print("Ring file %s is invalid: %r" % (ring_file, exc)) else: if builder_dict == ring_dict: print("Ring file %s is up-to-date" % ring_file) else: print("Ring file %s is obsolete" % ring_file) if builder.devs: balance_per_dev = builder._build_balance_per_dev() print( "Devices: id region zone ip address port " "replication ip replication port name " "weight partitions balance flags meta" ) for dev in builder._iter_devs(): flags = "DEL" if dev in builder._remove_devs else "" print( " %5d %7d %5d %15s %5d %15s %17d %9s %6.02f " "%10s %7.02f %5s %s" % ( dev["id"], dev["region"], dev["zone"], dev["ip"], dev["port"], dev["replication_ip"], dev["replication_port"], dev["device"], dev["weight"], dev["parts"], balance_per_dev[dev["id"]], flags, dev["meta"], ) ) exit(EXIT_SUCCESS)
def _fake_load(gz_path, stub_objs, metadata_only=False): return RingData(devs=stub_objs[os.path.basename(gz_path)[:-8]], replica2part2dev_id=[], part_shift=24)
from swift.common.ring import RingData, RingBuilder from array import array import math import pickle """ https://docs.openstack.org/swift/pike/admin/objectstorage-troubleshoot.html Using existing swift tools, there is no way to recover a builder file from a ring.gz file. However, if you have a knowledge of Python, it is possible to construct a builder file that is pretty close to the one you have lost. """ ring = RingData.load('/home/shahbazi/Desktop/rings/account.ring.gz') partitions = len(ring._replica2part2dev_id[0]) replicas = len(ring._replica2part2dev_id) builder = RingBuilder(int(math.log(partitions, 2)), replicas, 1) builder.devs = ring.devs builder._replica2part2dev = ring._replica2part2dev_id builder._last_part_moves_epoch = 0 builder._last_part_moves = array('B', (0 for _ in range(partitions))) builder.change_min_part_hours(24) # builder._set_parts_wanted()
def default(): """ swift-ring-builder <builder_file> Shows information about the ring and the devices within. Output includes a table that describes the report parameters (id, region, port, flags, etc). flags: possible values are 'DEL' and '' DEL - indicates that the device is marked for removal from ring and will be removed in next rebalance. """ try: builder_id = builder.id except AttributeError: builder_id = "(not assigned)" print('%s, build version %d, id %s' % (builder_file, builder.version, builder_id)) regions = 0 zones = 0 balance = 0 dev_count = 0 if builder.devs: regions = len( set(d['region'] for d in builder.devs if d is not None)) zones = len( set((d['region'], d['zone']) for d in builder.devs if d is not None)) dev_count = len([dev for dev in builder.devs if dev is not None]) balance = builder.get_balance() dispersion_trailer = '' if builder.dispersion is None else ( ', %.02f dispersion' % (builder.dispersion)) print('%d partitions, %.6f replicas, %d regions, %d zones, ' '%d devices, %.02f balance%s' % (builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer)) print('The minimum number of hours before a partition can be ' 'reassigned is %s (%s remaining)' % (builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left))) print('The overload factor is %0.2f%% (%.6f)' % (builder.overload * 100, builder.overload)) # compare ring file against builder file if not exists(ring_file): print('Ring file %s not found, ' 'probably it hasn\'t been written yet' % ring_file) else: builder_dict = builder.get_ring().to_dict() try: ring_dict = RingData.load(ring_file).to_dict() except Exception as exc: print('Ring file %s is invalid: %r' % (ring_file, exc)) else: if builder_dict == ring_dict: print('Ring file %s is up-to-date' % ring_file) else: print('Ring file %s is obsolete' % ring_file) if builder.devs: balance_per_dev = builder._build_balance_per_dev() header_line, print_dev_f = _make_display_device_table(builder) print(header_line) for dev in builder._iter_devs(): flags = 'DEL' if dev in builder._remove_devs else '' print_dev_f(dev, balance_per_dev[dev['id']], flags) exit(EXIT_SUCCESS)
def main(): usage = "usage: %prog [options] arg" parser = OptionParser(usage) group = OptionGroup(parser, "Moving Map Building") group.add_option("-o", "--old_ring", dest="old_ring_path", help="The path to the old ring file.") group.add_option("-n", "--new_ring", dest="new_ring_path", help="The path to the new ring file.") group.add_option("-t", "--test", dest="test", default="False", help="Controls whether the execution will only printout " "the commands to standard output, " "without explicitly run them. To set (True) " "to unset (False). The default value is False. " "Used for manual testing only.") group.add_option("-r", "--run", dest="run", default="False", help="Controls the data movement script execution. " " Default is False") group.add_option("-f", "--file", dest="dump_file", help="The path to the data moving map dump file.") group.add_option("-c", "--concurrency", dest="concurrency", type="int", default=1, help="The concurrency level, the default is 1.") group.add_option("-m", "--mover_tmp_dir", dest="mover_tmp_dir", default='data_mover', help="The name of temporal directory that would " "be used for data migration") parser.add_option_group(group) (options, args) = parser.parse_args() validate_moving_map_options(options, args) if options.dump_file: dump_file = options.dump_file else: dump_file = DEFAULT_DUMP_FILE if bool(options.run): conf = {} conf['data_moving_map_dump'] = dump_file conf['concurrency'] = options.concurrency conf['test'] = options.test conf['mover_tmp_dir'] = options.mover_tmp_dir objMover = ObjectMover(conf=conf) objMover.run_once() else: old_ring_data = RingData.load(options.old_ring_path) new_ring_data = RingData.load(options.new_ring_path) if bool(options.test): print_moving_map(old_ring_data, new_ring_data, 10) dump_moving_map(old_ring_data, new_ring_data, dump_file) if bool(options.test): load_moving_map(dump_file, True)