def default():
"""
swift-ring-builder <builder_file>
Shows information about the ring and the devices within. Output
includes a table that describes the report parameters (id, region,
port, flags, etc).
flags: possible values are 'DEL' and ''
DEL - indicates that the device is marked for removal from
ring and will be removed in next rebalance.
"""
print('%s, build version %d' % (builder_file, builder.version))
regions = 0
zones = 0
balance = 0
dev_count = 0
if builder.devs:
regions = len(set(d['region'] for d in builder.devs
if d is not None))
zones = len(set((d['region'], d['zone']) for d in builder.devs
if d is not None))
dev_count = len([dev for dev in builder.devs
if dev is not None])
balance = builder.get_balance()
dispersion_trailer = '' if builder.dispersion is None else (
', %.02f dispersion' % (builder.dispersion))
print('%d partitions, %.6f replicas, %d regions, %d zones, '
'%d devices, %.02f balance%s' % (
builder.parts, builder.replicas, regions, zones, dev_count,
balance, dispersion_trailer))
print('The minimum number of hours before a partition can be '
'reassigned is %s (%s remaining)' % (
builder.min_part_hours,
timedelta(seconds=builder.min_part_seconds_left)))
print('The overload factor is %0.2f%% (%.6f)' % (
builder.overload * 100, builder.overload))
# compare ring file against builder file
if not exists(ring_file):
print('Ring file %s not found, '
'probably it hasn\'t been written yet' % ring_file)
else:
builder_dict = builder.get_ring().to_dict()
try:
ring_dict = RingData.load(ring_file).to_dict()
except Exception as exc:
print('Ring file %s is invalid: %r' % (ring_file, exc))
else:
if builder_dict == ring_dict:
print('Ring file %s is up-to-date' % ring_file)
else:
print('Ring file %s is obsolete' % ring_file)
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
header_line, print_dev_f = _make_display_device_table(builder)
print(header_line)
for dev in builder._iter_devs():
flags = 'DEL' if dev in builder._remove_devs else ''
print_dev_f(dev, balance_per_dev[dev['id']], flags)
exit(EXIT_SUCCESS)
def setUp(self):
utils.HASH_PATH_SUFFIX = b'endcap'
utils.HASH_PATH_PREFIX = b''
self.testdir = mkdtemp()
ring_file = os.path.join(self.testdir, 'container.ring.gz')
with closing(GzipFile(ring_file, 'wb')) as f:
pickle.dump(
RingData([[0, 1, 2, 0, 1, 2], [1, 2, 0, 1, 2, 0],
[2, 3, 1, 2, 3, 1]], [{
'id': 0,
'ip': '127.0.0.1',
'port': 1,
'device': 'sda1',
'zone': 0
}, {
'id': 1,
'ip': '127.0.0.1',
'port': 1,
'device': 'sda1',
'zone': 2
}, {
'id': 2,
'ip': '127.0.0.1',
'port': 1,
'device': 'sda1',
'zone': 4
}, {
'id': 3,
'ip': '127.0.0.1',
'port': 1,
'device': 'sda1',
'zone': 6
}], 30), f)
self.devices_dir = os.path.join(self.testdir, 'devices')
os.mkdir(self.devices_dir)
self.sda1 = os.path.join(self.devices_dir, 'sda1')
os.mkdir(self.sda1)
for policy in POLICIES:
os.mkdir(os.path.join(self.sda1, get_tmp_dir(policy)))
self.logger = debug_logger()
self.ts_iter = make_timestamp_iter()
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = ''
self.testdir = os.path.join(os.path.dirname(__file__),
'object_updater')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
ring_file = os.path.join(self.testdir, 'container.ring.gz')
with closing(GzipFile(ring_file, 'wb')) as f:
pickle.dump(
RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'ip': '127.0.0.1', 'port': 1,
'device': 'sda1', 'zone': 0},
{'id': 1, 'ip': '127.0.0.1', 'port': 1,
'device': 'sda1', 'zone': 2}], 30),
f)
self.devices_dir = os.path.join(self.testdir, 'devices')
os.mkdir(self.devices_dir)
self.sda1 = os.path.join(self.devices_dir, 'sda1')
os.mkdir(self.sda1)
os.mkdir(os.path.join(self.sda1, 'tmp'))
def all_bind_ports_for_node(self):
"""
Given an iterable of IP addresses identifying a storage backend server,
return a set of all bind ports defined in all rings for this storage
backend server.
The caller is responsible for not calling this method (which performs
at least a stat on all ring files) too frequently.
"""
# NOTE: we don't worry about disappearing rings here because you can't
# ever delete a storage policy.
for policy in POLICIES:
# NOTE: we must NOT use policy.load_ring to load the ring. Users
# of this utility function will not need the actual ring data, just
# the bind ports.
#
# This is duplicated with Ring.__init__ just a bit...
serialized_path = os.path.join(self.swift_dir,
policy.ring_name + '.ring.gz')
try:
new_mtime = os.path.getmtime(serialized_path)
except OSError:
continue
old_mtime = self.mtimes_by_ring_path.get(serialized_path)
if not old_mtime or old_mtime != new_mtime:
self.portsets_by_ring_path[serialized_path] = set(
dev['port']
for dev in RingData.load(serialized_path,
metadata_only=True).devs
if dev and dev['ip'] in self.my_ips)
self.mtimes_by_ring_path[serialized_path] = new_mtime
# No "break" here so that the above line will update the
# mtimes_by_ring_path entry for any ring that changes, not just
# the first one we notice.
# Return the requested set of ports from our (now-freshened) cache
return six.moves.reduce(set.union,
self.portsets_by_ring_path.values(), set())
def all_bind_ports_for_node(self):
"""
Given an iterable of IP addresses identifying a storage backend server,
return a set of all bind ports defined in all rings for this storage
backend server.
The caller is responsible for not calling this method (which performs
at least a stat on all ring files) too frequently.
"""
# NOTE: we don't worry about disappearing rings here because you can't
# ever delete a storage policy.
for policy in POLICIES:
# NOTE: we must NOT use policy.load_ring to load the ring. Users
# of this utility function will not need the actual ring data, just
# the bind ports.
#
# This is duplicated with Ring.__init__ just a bit...
serialized_path = os.path.join(self.swift_dir,
policy.ring_name + '.ring.gz')
try:
new_mtime = os.path.getmtime(serialized_path)
except OSError:
continue
old_mtime = self.mtimes_by_ring_path.get(serialized_path)
if not old_mtime or old_mtime != new_mtime:
self.portsets_by_ring_path[serialized_path] = set(
dev['port']
for dev in RingData.load(serialized_path,
metadata_only=True).devs
if dev and dev['ip'] in self.my_ips)
self.mtimes_by_ring_path[serialized_path] = new_mtime
# No "break" here so that the above line will update the
# mtimes_by_ring_path entry for any ring that changes, not just
# the first one we notice.
# Return the requested set of ports from our (now-freshened) cache
return six.moves.reduce(set.union, self.portsets_by_ring_path.values(),
set())
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
self.testdir = os.path.join(mkdtemp(), 'tmp_test_container_updater')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
pickle.dump(
RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{
'id': 0,
'ip': '127.0.0.1',
'port': 12345,
'device': 'sda1',
'zone': 0
}, {
'id': 1,
'ip': '127.0.0.1',
'port': 12345,
'device': 'sda1',
'zone': 2
}], 30),
GzipFile(os.path.join(self.testdir, 'account.ring.gz'), 'wb'))
self.devices_dir = os.path.join(self.testdir, 'devices')
os.mkdir(self.devices_dir)
self.sda1 = os.path.join(self.devices_dir, 'sda1')
os.mkdir(self.sda1)
def default():
"""
swift-ring-builder <builder_file>
Shows information about the ring and the devices within.
Flags:
DEL - marked for removal and will be removed next rebalance.
"""
print('%s, build version %d' % (builder_file, builder.version))
regions = 0
zones = 0
balance = 0
dev_count = 0
if builder.devs:
regions = len(set(d['region'] for d in builder.devs
if d is not None))
zones = len(set((d['region'], d['zone']) for d in builder.devs
if d is not None))
dev_count = len([dev for dev in builder.devs
if dev is not None])
balance = builder.get_balance()
dispersion_trailer = '' if builder.dispersion is None else (
', %.02f dispersion' % (builder.dispersion))
print('%d partitions, %.6f replicas, %d regions, %d zones, '
'%d devices, %.02f balance%s' % (
builder.parts, builder.replicas, regions, zones, dev_count,
balance, dispersion_trailer))
print('The minimum number of hours before a partition can be '
'reassigned is %s (%s remaining)' % (
builder.min_part_hours,
timedelta(seconds=builder.min_part_seconds_left)))
print('The overload factor is %0.2f%% (%.6f)' % (
builder.overload * 100, builder.overload))
# compare ring file against builder file
if not exists(ring_file):
print('Ring file %s not found, '
'probably it hasn\'t been written yet' % ring_file)
else:
builder_dict = builder.get_ring().to_dict()
try:
ring_dict = RingData.load(ring_file).to_dict()
except Exception as exc:
print('Ring file %s is invalid: %r' % (ring_file, exc))
else:
if builder_dict == ring_dict:
print('Ring file %s is up-to-date' % ring_file)
else:
print('Ring file %s is obsolete' % ring_file)
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
print('Devices: id region zone ip address port '
'replication ip replication port name '
'weight partitions balance flags meta')
for dev in builder._iter_devs():
flags = 'DEL' if dev in builder._remove_devs else ''
print(' %5d %7d %5d %15s %5d %15s %17d %9s %6.02f '
'%10s %7.02f %5s %s' %
(dev['id'], dev['region'], dev['zone'], dev['ip'],
dev['port'], dev['replication_ip'],
dev['replication_port'], dev['device'], dev['weight'],
dev['parts'], balance_per_dev[dev['id']], flags,
dev['meta']))
exit(EXIT_SUCCESS)
def setup_servers(the_object_server=object_server, extra_conf=None):
"""
Setup proxy, account, container and object servers using a set of fake
rings and policies.
:param the_object_server: The object server module to use (optional,
defaults to swift.obj.server)
:param extra_conf: A dict of config options that will update the basic
config passed to all server instances.
:returns: A dict containing the following entries:
orig_POLICIES: the value of storage_policy.POLICIES prior to
it being patched with fake policies
orig_SysLogHandler: the value of utils.SysLogHandler prior to
it being patched
testdir: root directory used for test files
test_POLICIES: a StoragePolicyCollection of fake policies
test_servers: a tuple of test server instances
test_sockets: a tuple of sockets used by test servers
test_coros: a tuple of greenthreads in which test servers are
running
"""
context = {
"orig_POLICIES": storage_policy._POLICIES,
"orig_SysLogHandler": utils.SysLogHandler}
utils.HASH_PATH_SUFFIX = b'endcap'
utils.SysLogHandler = mock.MagicMock()
# Since we're starting up a lot here, we're going to test more than
# just chunked puts; we're also going to test parts of
# proxy_server.Application we couldn't get to easily otherwise.
context["testdir"] = _testdir = \
os.path.join(mkdtemp(), 'tmp_test_proxy_server_chunked')
mkdirs(_testdir)
rmtree(_testdir)
for drive in ('sda1', 'sdb1', 'sdc1', 'sdd1', 'sde1',
'sdf1', 'sdg1', 'sdh1', 'sdi1', 'sdj1',
'sdk1', 'sdl1'):
mkdirs(os.path.join(_testdir, drive, 'tmp'))
conf = {'devices': _testdir, 'swift_dir': _testdir,
'mount_check': 'false', 'allowed_headers':
'content-encoding, x-object-manifest, content-disposition, foo',
'allow_versions': 't'}
if extra_conf:
conf.update(extra_conf)
prolis = listen_zero()
acc1lis = listen_zero()
acc2lis = listen_zero()
con1lis = listen_zero()
con2lis = listen_zero()
obj1lis = listen_zero()
obj2lis = listen_zero()
obj3lis = listen_zero()
obj4lis = listen_zero()
obj5lis = listen_zero()
obj6lis = listen_zero()
objsocks = [obj1lis, obj2lis, obj3lis, obj4lis, obj5lis, obj6lis]
context["test_sockets"] = \
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis, obj3lis,
obj4lis, obj5lis, obj6lis)
account_ring_path = os.path.join(_testdir, 'account.ring.gz')
account_devs = [
{'port': acc1lis.getsockname()[1]},
{'port': acc2lis.getsockname()[1]},
]
write_fake_ring(account_ring_path, *account_devs)
container_ring_path = os.path.join(_testdir, 'container.ring.gz')
container_devs = [
{'port': con1lis.getsockname()[1]},
{'port': con2lis.getsockname()[1]},
]
write_fake_ring(container_ring_path, *container_devs)
storage_policy._POLICIES = storage_policy.StoragePolicyCollection([
StoragePolicy(0, 'zero', True),
StoragePolicy(1, 'one', False),
StoragePolicy(2, 'two', False),
ECStoragePolicy(3, 'ec', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=2, ec_nparity=1, ec_segment_size=4096),
ECStoragePolicy(4, 'ec-dup', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=2, ec_nparity=1, ec_segment_size=4096,
ec_duplication_factor=2)])
obj_rings = {
0: ('sda1', 'sdb1'),
1: ('sdc1', 'sdd1'),
2: ('sde1', 'sdf1'),
# sdg1, sdh1, sdi1 taken by policy 3 (see below)
}
for policy_index, devices in obj_rings.items():
policy = storage_policy.POLICIES[policy_index]
obj_ring_path = os.path.join(_testdir, policy.ring_name + '.ring.gz')
obj_devs = [
{'port': objsock.getsockname()[1], 'device': dev}
for objsock, dev in zip(objsocks, devices)]
write_fake_ring(obj_ring_path, *obj_devs)
# write_fake_ring can't handle a 3-element ring, and the EC policy needs
# at least 6 devs to work with (ec_k=2, ec_m=1, duplication_factor=2),
# so we do it manually
devs = [{'id': 0, 'zone': 0, 'device': 'sdg1', 'ip': '127.0.0.1',
'port': obj1lis.getsockname()[1]},
{'id': 1, 'zone': 0, 'device': 'sdh1', 'ip': '127.0.0.1',
'port': obj2lis.getsockname()[1]},
{'id': 2, 'zone': 0, 'device': 'sdi1', 'ip': '127.0.0.1',
'port': obj3lis.getsockname()[1]},
{'id': 3, 'zone': 0, 'device': 'sdj1', 'ip': '127.0.0.1',
'port': obj4lis.getsockname()[1]},
{'id': 4, 'zone': 0, 'device': 'sdk1', 'ip': '127.0.0.1',
'port': obj5lis.getsockname()[1]},
{'id': 5, 'zone': 0, 'device': 'sdl1', 'ip': '127.0.0.1',
'port': obj6lis.getsockname()[1]}]
pol3_replica2part2dev_id = [[0, 1, 2, 0],
[1, 2, 0, 1],
[2, 0, 1, 2]]
pol4_replica2part2dev_id = [[0, 1, 2, 3],
[1, 2, 3, 4],
[2, 3, 4, 5],
[3, 4, 5, 0],
[4, 5, 0, 1],
[5, 0, 1, 2]]
obj3_ring_path = os.path.join(
_testdir, storage_policy.POLICIES[3].ring_name + '.ring.gz')
part_shift = 30
with closing(GzipFile(obj3_ring_path, 'wb')) as fh:
pickle.dump(RingData(pol3_replica2part2dev_id, devs, part_shift), fh)
obj4_ring_path = os.path.join(
_testdir, storage_policy.POLICIES[4].ring_name + '.ring.gz')
part_shift = 30
with closing(GzipFile(obj4_ring_path, 'wb')) as fh:
pickle.dump(RingData(pol4_replica2part2dev_id, devs, part_shift), fh)
prosrv = proxy_server.Application(conf, logger=debug_logger('proxy'))
for policy in storage_policy.POLICIES:
# make sure all the rings are loaded
prosrv.get_object_ring(policy.idx)
# don't lose this one!
context["test_POLICIES"] = storage_policy._POLICIES
acc1srv = account_server.AccountController(
conf, logger=debug_logger('acct1'))
acc2srv = account_server.AccountController(
conf, logger=debug_logger('acct2'))
con1srv = container_server.ContainerController(
conf, logger=debug_logger('cont1'))
con2srv = container_server.ContainerController(
conf, logger=debug_logger('cont2'))
obj1srv = the_object_server.ObjectController(
conf, logger=debug_logger('obj1'))
obj2srv = the_object_server.ObjectController(
conf, logger=debug_logger('obj2'))
obj3srv = the_object_server.ObjectController(
conf, logger=debug_logger('obj3'))
obj4srv = the_object_server.ObjectController(
conf, logger=debug_logger('obj4'))
obj5srv = the_object_server.ObjectController(
conf, logger=debug_logger('obj5'))
obj6srv = the_object_server.ObjectController(
conf, logger=debug_logger('obj6'))
context["test_servers"] = \
(prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv, obj3srv,
obj4srv, obj5srv, obj6srv)
nl = NullLogger()
logging_prosv = proxy_logging.ProxyLoggingMiddleware(
listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger)
prospa = spawn(wsgi.server, prolis, logging_prosv, nl)
acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl)
acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl)
con1spa = spawn(wsgi.server, con1lis, con1srv, nl)
con2spa = spawn(wsgi.server, con2lis, con2srv, nl)
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl)
obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl)
obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl)
obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl)
context["test_coros"] = \
(prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa,
obj4spa, obj5spa, obj6spa)
# Create account
ts = normalize_timestamp(time.time())
partition, nodes = prosrv.account_ring.get_nodes('a')
for node in nodes:
conn = swift.proxy.controllers.obj.http_connect(node['ip'],
node['port'],
node['device'],
partition, 'PUT', '/a',
{'X-Timestamp': ts,
'x-trans-id': 'test'})
resp = conn.getresponse()
assert(resp.status == 201)
# Create another account
# used for account-to-account tests
ts = normalize_timestamp(time.time())
partition, nodes = prosrv.account_ring.get_nodes('a1')
for node in nodes:
conn = swift.proxy.controllers.obj.http_connect(node['ip'],
node['port'],
node['device'],
partition, 'PUT',
'/a1',
{'X-Timestamp': ts,
'x-trans-id': 'test'})
resp = conn.getresponse()
assert(resp.status == 201)
# Create containers, 1 per test policy
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nX-Auth-Token: t\r\n'
'Content-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % (
exp, headers[:len(exp)])
# Create container in other account
# used for account-to-account tests
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a1/c1 HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nX-Auth-Token: t\r\n'
'Content-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % (
exp, headers[:len(exp)])
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write(
'PUT /v1/a/c1 HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: one\r\n'
'Content-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
assert headers[:len(exp)] == exp, \
"Expected '%s', encountered '%s'" % (exp, headers[:len(exp)])
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write(
'PUT /v1/a/c2 HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nX-Auth-Token: t\r\nX-Storage-Policy: two\r\n'
'Content-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
assert headers[:len(exp)] == exp, \
"Expected '%s', encountered '%s'" % (exp, headers[:len(exp)])
return context
def default():
"""
swift-ring-builder <builder_file>
Shows information about the ring and the devices within.
Flags:
DEL - marked for removal and will be removed next rebalance.
"""
print('%s, build version %d' % (builder_file, builder.version))
regions = 0
zones = 0
balance = 0
dev_count = 0
if builder.devs:
regions = len(
set(d['region'] for d in builder.devs if d is not None))
zones = len(
set((d['region'], d['zone']) for d in builder.devs
if d is not None))
dev_count = len([dev for dev in builder.devs if dev is not None])
balance = builder.get_balance()
dispersion_trailer = '' if builder.dispersion is None else (
', %.02f dispersion' % (builder.dispersion))
print('%d partitions, %.6f replicas, %d regions, %d zones, '
'%d devices, %.02f balance%s' %
(builder.parts, builder.replicas, regions, zones, dev_count,
balance, dispersion_trailer))
print('The minimum number of hours before a partition can be '
'reassigned is %s (%s remaining)' %
(builder.min_part_hours,
timedelta(seconds=builder.min_part_seconds_left)))
print('The overload factor is %0.2f%% (%.6f)' %
(builder.overload * 100, builder.overload))
# compare ring file against builder file
if not exists(ring_file):
print('Ring file %s not found, '
'probably it hasn\'t been written yet' % ring_file)
else:
builder_dict = builder.get_ring().to_dict()
try:
ring_dict = RingData.load(ring_file).to_dict()
except Exception as exc:
print('Ring file %s is invalid: %r' % (ring_file, exc))
else:
if builder_dict == ring_dict:
print('Ring file %s is up-to-date' % ring_file)
else:
print('Ring file %s is obsolete' % ring_file)
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
print('Devices: id region zone ip address port '
'replication ip replication port name '
'weight partitions balance flags meta')
for dev in builder._iter_devs():
flags = 'DEL' if dev in builder._remove_devs else ''
print(' %5d %7d %5d %15s %5d %15s %17d %9s %6.02f '
'%10s %7.02f %5s %s' %
(dev['id'], dev['region'], dev['zone'], dev['ip'],
dev['port'], dev['replication_ip'],
dev['replication_port'], dev['device'], dev['weight'],
dev['parts'], balance_per_dev[dev['id']], flags,
dev['meta']))
exit(EXIT_SUCCESS)
def default():
"""
swift-ring-builder <builder_file>
Shows information about the ring and the devices within.
Flags:
DEL - marked for removal and will be removed next rebalance.
"""
print("%s, build version %d" % (builder_file, builder.version))
regions = 0
zones = 0
balance = 0
dev_count = 0
if builder.devs:
regions = len(set(d["region"] for d in builder.devs if d is not None))
zones = len(set((d["region"], d["zone"]) for d in builder.devs if d is not None))
dev_count = len([dev for dev in builder.devs if dev is not None])
balance = builder.get_balance()
dispersion_trailer = "" if builder.dispersion is None else (", %.02f dispersion" % (builder.dispersion))
print(
"%d partitions, %.6f replicas, %d regions, %d zones, "
"%d devices, %.02f balance%s"
% (builder.parts, builder.replicas, regions, zones, dev_count, balance, dispersion_trailer)
)
print(
"The minimum number of hours before a partition can be "
"reassigned is %s (%s remaining)"
% (builder.min_part_hours, timedelta(seconds=builder.min_part_seconds_left))
)
print("The overload factor is %0.2f%% (%.6f)" % (builder.overload * 100, builder.overload))
# compare ring file against builder file
if not exists(ring_file):
print("Ring file %s not found, " "probably it hasn't been written yet" % ring_file)
else:
builder_dict = builder.get_ring().to_dict()
try:
ring_dict = RingData.load(ring_file).to_dict()
except Exception as exc:
print("Ring file %s is invalid: %r" % (ring_file, exc))
else:
if builder_dict == ring_dict:
print("Ring file %s is up-to-date" % ring_file)
else:
print("Ring file %s is obsolete" % ring_file)
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
print(
"Devices: id region zone ip address port "
"replication ip replication port name "
"weight partitions balance flags meta"
)
for dev in builder._iter_devs():
flags = "DEL" if dev in builder._remove_devs else ""
print(
" %5d %7d %5d %15s %5d %15s %17d %9s %6.02f "
"%10s %7.02f %5s %s"
% (
dev["id"],
dev["region"],
dev["zone"],
dev["ip"],
dev["port"],
dev["replication_ip"],
dev["replication_port"],
dev["device"],
dev["weight"],
dev["parts"],
balance_per_dev[dev["id"]],
flags,
dev["meta"],
)
)
exit(EXIT_SUCCESS)
def _fake_load(gz_path, stub_objs, metadata_only=False):
return RingData(devs=stub_objs[os.path.basename(gz_path)[:-8]],
replica2part2dev_id=[],
part_shift=24)
from swift.common.ring import RingData, RingBuilder
from array import array
import math
import pickle
"""
https://docs.openstack.org/swift/pike/admin/objectstorage-troubleshoot.html
Using existing swift tools, there is no way to recover a builder file from a ring.gz file. However,
if you have a knowledge of Python, it is possible to construct a builder file that is pretty close to the one you have lost.
"""
ring = RingData.load('/home/shahbazi/Desktop/rings/account.ring.gz')
partitions = len(ring._replica2part2dev_id[0])
replicas = len(ring._replica2part2dev_id)
builder = RingBuilder(int(math.log(partitions, 2)), replicas, 1)
builder.devs = ring.devs
builder._replica2part2dev = ring._replica2part2dev_id
builder._last_part_moves_epoch = 0
builder._last_part_moves = array('B', (0 for _ in range(partitions)))
builder.change_min_part_hours(24)
# builder._set_parts_wanted()
def default():
"""
swift-ring-builder <builder_file>
Shows information about the ring and the devices within. Output
includes a table that describes the report parameters (id, region,
port, flags, etc).
flags: possible values are 'DEL' and ''
DEL - indicates that the device is marked for removal from
ring and will be removed in next rebalance.
"""
try:
builder_id = builder.id
except AttributeError:
builder_id = "(not assigned)"
print('%s, build version %d, id %s' %
(builder_file, builder.version, builder_id))
regions = 0
zones = 0
balance = 0
dev_count = 0
if builder.devs:
regions = len(
set(d['region'] for d in builder.devs if d is not None))
zones = len(
set((d['region'], d['zone']) for d in builder.devs
if d is not None))
dev_count = len([dev for dev in builder.devs if dev is not None])
balance = builder.get_balance()
dispersion_trailer = '' if builder.dispersion is None else (
', %.02f dispersion' % (builder.dispersion))
print('%d partitions, %.6f replicas, %d regions, %d zones, '
'%d devices, %.02f balance%s' %
(builder.parts, builder.replicas, regions, zones, dev_count,
balance, dispersion_trailer))
print('The minimum number of hours before a partition can be '
'reassigned is %s (%s remaining)' %
(builder.min_part_hours,
timedelta(seconds=builder.min_part_seconds_left)))
print('The overload factor is %0.2f%% (%.6f)' %
(builder.overload * 100, builder.overload))
# compare ring file against builder file
if not exists(ring_file):
print('Ring file %s not found, '
'probably it hasn\'t been written yet' % ring_file)
else:
builder_dict = builder.get_ring().to_dict()
try:
ring_dict = RingData.load(ring_file).to_dict()
except Exception as exc:
print('Ring file %s is invalid: %r' % (ring_file, exc))
else:
if builder_dict == ring_dict:
print('Ring file %s is up-to-date' % ring_file)
else:
print('Ring file %s is obsolete' % ring_file)
if builder.devs:
balance_per_dev = builder._build_balance_per_dev()
header_line, print_dev_f = _make_display_device_table(builder)
print(header_line)
for dev in builder._iter_devs():
flags = 'DEL' if dev in builder._remove_devs else ''
print_dev_f(dev, balance_per_dev[dev['id']], flags)
exit(EXIT_SUCCESS)
def main():
usage = "usage: %prog [options] arg"
parser = OptionParser(usage)
group = OptionGroup(parser, "Moving Map Building")
group.add_option("-o",
"--old_ring",
dest="old_ring_path",
help="The path to the old ring file.")
group.add_option("-n",
"--new_ring",
dest="new_ring_path",
help="The path to the new ring file.")
group.add_option("-t",
"--test",
dest="test",
default="False",
help="Controls whether the execution will only printout "
"the commands to standard output, "
"without explicitly run them. To set (True) "
"to unset (False). The default value is False. "
"Used for manual testing only.")
group.add_option("-r",
"--run",
dest="run",
default="False",
help="Controls the data movement script execution. "
" Default is False")
group.add_option("-f",
"--file",
dest="dump_file",
help="The path to the data moving map dump file.")
group.add_option("-c",
"--concurrency",
dest="concurrency",
type="int",
default=1,
help="The concurrency level, the default is 1.")
group.add_option("-m",
"--mover_tmp_dir",
dest="mover_tmp_dir",
default='data_mover',
help="The name of temporal directory that would "
"be used for data migration")
parser.add_option_group(group)
(options, args) = parser.parse_args()
validate_moving_map_options(options, args)
if options.dump_file:
dump_file = options.dump_file
else:
dump_file = DEFAULT_DUMP_FILE
if bool(options.run):
conf = {}
conf['data_moving_map_dump'] = dump_file
conf['concurrency'] = options.concurrency
conf['test'] = options.test
conf['mover_tmp_dir'] = options.mover_tmp_dir
objMover = ObjectMover(conf=conf)
objMover.run_once()
else:
old_ring_data = RingData.load(options.old_ring_path)
new_ring_data = RingData.load(options.new_ring_path)
if bool(options.test):
print_moving_map(old_ring_data, new_ring_data, 10)
dump_moving_map(old_ring_data, new_ring_data, dump_file)
if bool(options.test):
load_moving_map(dump_file, True)
def main():
usage = "usage: %prog [options] arg"
parser = OptionParser(usage)
group = OptionGroup(parser, "Moving Map Building")
group.add_option("-o", "--old_ring", dest="old_ring_path",
help="The path to the old ring file.")
group.add_option("-n", "--new_ring", dest="new_ring_path",
help="The path to the new ring file.")
group.add_option("-t", "--test", dest="test", default="False",
help="Controls whether the execution will only printout "
"the commands to standard output, "
"without explicitly run them. To set (True) "
"to unset (False). The default value is False. "
"Used for manual testing only.")
group.add_option("-r", "--run", dest="run", default="False",
help="Controls the data movement script execution. "
" Default is False")
group.add_option("-f", "--file", dest="dump_file",
help="The path to the data moving map dump file.")
group.add_option("-c", "--concurrency", dest="concurrency", type="int",
default=1,
help="The concurrency level, the default is 1.")
group.add_option("-m", "--mover_tmp_dir", dest="mover_tmp_dir",
default='data_mover',
help="The name of temporal directory that would "
"be used for data migration")
parser.add_option_group(group)
(options, args) = parser.parse_args()
validate_moving_map_options(options, args)
if options.dump_file:
dump_file = options.dump_file
else:
dump_file = DEFAULT_DUMP_FILE
if bool(options.run):
conf = {}
conf['data_moving_map_dump'] = dump_file
conf['concurrency'] = options.concurrency
conf['test'] = options.test
conf['mover_tmp_dir'] = options.mover_tmp_dir
objMover = ObjectMover(conf=conf)
objMover.run_once()
else:
old_ring_data = RingData.load(options.old_ring_path)
new_ring_data = RingData.load(options.new_ring_path)
if bool(options.test):
print_moving_map(old_ring_data, new_ring_data, 10)
dump_moving_map(old_ring_data, new_ring_data, dump_file)
if bool(options.test):
load_moving_map(dump_file, True)
