def reweight_by_utilization(options):
if options.oload <= 100:
raise Exception("You must give a percentage higher than 100.")
cephinfo.init_pg()
pgm = cephinfo.pg_data
pgs_by_osd = defaultdict(int)
if options.by_pg:
weight_sum = 0.0
num_pg_copies = 0
num_osds = 0
for p in pgm['pg_stats']:
pool = p['pgid'].split('.')[0]
if options.pools and pool not in options.pools:
continue
for q in p['up']:
if not pgs_by_osd[q]:
pgs_by_osd[q] = 0
weight_sum += get_weight(q, 'crush_weight')
num_osds += 1
pgs_by_osd[q] += 1
num_pg_copies += 1
if not num_osds or (num_pg_copies / num_osds <
mon_reweight_min_pgs_per_osd):
raise Exception(
'Refusing to reweight: we only have %d PGs across %d osds!' %
(num_pg_copies, num_osds))
average_util = num_pg_copies / weight_sum
if VERBOSE:
print "weight_sum: %3f, num_pg_copies: %d, num_osds: %d" % (
weight_sum, num_pg_copies, num_osds)
else:
num_osd = len(pgm['osd_stats'])
if num_osd < MIN_OSDS:
raise Exception(
"Refusing to reweight: we have only %d OSDs! (%d needed)" %
(num_osd, MIN_OSDS))
# Avoid putting a small number (or 0) in the denominator when calculating average_util
if pgm['osd_stats_sum'][
'kb'] * 1024 / num_osd < mon_reweight_min_bytes_per_osd:
raise Exception(
"Refusing to reweight: we only have %d GB total space across all osds! (%d GB needed)"
% (pgm['osd_stats_sum']['kb'] / 1024 / 1024,
mon_reweight_min_bytes_per_osd * num_osd / 1024 / 1024 /
1024))
if pgm['osd_stats_sum'][
'kb_used'] * 1024 / num_osd < mon_reweight_min_bytes_used_per_osd:
raise Exception(
"Refusing to reweight: we only have %d GB used across all osds! (%d GB needed)"
% (pgm['osd_stats_sum']['kb_used'] / 1024 / 1024,
mon_reweight_min_bytes_used_per_osd * num_osd / 1024 /
1024 / 1024))
average_util = float(pgm['osd_stats_sum']['kb_used']) / float(
pgm['osd_stats_sum']['kb'])
if VERBOSE: print "Found %d OSDs in total" % len(pgm['osd_stats'])
# filter out the empty osds
nonempty_osds = [
osd for osd in pgm['osd_stats'] if float(osd['kb']) > 0
and get_weight(osd['osd'], type='crush_weight') > 0
]
if VERBOSE: print "Found %d non-empty OSDs" % len(nonempty_osds)
# optionally filter out osds not in the requested bucket
# and recalculate average_util
if options.bucket:
bucket_osds = []
for bucket in options.bucket:
bucket_osds.extend(ceph_osds_in_bucket.list(bucket))
sum_kb = 0
sum_weight = 0
sum_kb_used = 0
filtered_osds = []
for osd in nonempty_osds:
name = 'osd.%d' % osd['osd']
if name in bucket_osds:
sum_weight += get_weight(osd['osd'],
'crush_weight') * 1024 * 1024 * 1024
sum_kb_used += osd['kb_used']
filtered_osds.insert(0, osd)
average_util = float(sum_kb_used) / float(sum_weight)
if VERBOSE:
print "Found %d OSDs after filtering by bucket" % len(
filtered_osds)
else:
filtered_osds = nonempty_osds
# sort osds from most to least deviant from the average_util
if options.by_pg:
osds = sorted(
filtered_osds,
key=lambda osd: -abs(average_util - pgs_by_osd[osd['osd']] /
get_weight(osd['osd'], type='crush_weight')))
else:
#osds = sorted(filtered_osds, key=lambda osd: -abs(average_util - float(osd['kb_used']) / float(osd['kb'])))
osds = sorted(
filtered_osds,
key=lambda osd: -abs(average_util - float(osd['kb_used']) /
(get_weight(osd['osd'], type='crush_weight') *
1024 * 1024 * 1024)))
# adjust down only if we are above the threshold
overload_util = average_util * options.oload / 100.0
# but aggressively adjust weights up whenever possible
underload_util = average_util
if VERBOSE:
print "average_util: %04f, overload_util: %04f, underload_util: %04f. " % (
average_util, overload_util, underload_util)
n = 0
for osd in osds:
if options.by_pg:
util = pgs_by_osd[osd['osd']] / get_weight(osd['osd'],
type='crush_weight')
else:
util = float(osd['kb_used']) / (get_weight(
osd['osd'], type='crush_weight') * 1024 * 1024 * 1024)
# skip very empty OSDs
if util < 0.001:
continue
if util >= overload_util:
# Assign a lower weight to overloaded OSDs. The current weight
# is a factor to take into account the original weights,
# to represent e.g. differing storage capacities
weight = get_weight(osd['osd'])
new_weight = (average_util / util) * float(weight)
new_weight = max(new_weight, weight - options.max_change)
print "osd.%d (%4f >= %4f) [%04f -> %04f]" % (
osd['osd'], util, overload_util, weight, new_weight)
if options.doit:
change_weight(osd['osd'], new_weight, options.really)
n += 1
if n >= options.num_osds: break
if not options.no_increasing and util <= underload_util:
# assign a higher weight.. if we can
weight = get_weight(osd['osd'])
new_weight = (average_util / util) * float(weight)
new_weight = max(new_weight, weight + 0.01)
new_weight = min(new_weight, weight + options.max_change)
if new_weight > 1.0:
new_weight = 1.0
if new_weight > weight:
print "osd.%d (%4f <= %4f) [%04f -> %04f]" % (
osd['osd'], util, underload_util, weight, new_weight)
if options.doit:
change_weight(osd['osd'], new_weight, options.really)
n += 1
if n >= options.num_osds: break
def reweight_by_utilization(options):
if options.oload <= 100:
raise Exception("You must give a percentage higher than 100.")
# "The reweighting threshold will be calculated as <average-utilization> "
# "times <input-percentage>. For example, an argument of 200 would "
# "reweight OSDs which are twice as utilized as the average OSD.\n";
cephinfo.init_pg()
pgm = cephinfo.pg_data
pgs_by_osd = defaultdict(int)
if options.by_pg:
weight_sum = 0.0
num_pg_copies = 0
num_osds = 0
for p in pgm['pg_stats']:
pool = p['pgid'].split('.')[0]
if options.pools and pool not in options.pools:
continue
for a in p['up']:
if not pgs_by_osd[a]:
num_osds += 1
weight_sum += get_weight(a,'crush_weight')
pgs_by_osd[a] = 0
pgs_by_osd[a] += 1
num_pg_copies += 1
if not num_osds or (num_pg_copies / num_osds < 10):
raise Exception('Refusing to reweight: we only have %d PGs across %d osds!' % (num_pg_copies, num_osds))
average_util = num_pg_copies / weight_sum
print "weight_sum: %3f, num_pg_copies: %d, num_osds: %d" % (weight_sum, num_pg_copies, num_osds)
else:
# Avoid putting a small number (or 0) in the denominator when calculating
# average_util
if pgm['osd_stats_sum']['kb'] < 1024:
raise Exception("Refusing to reweight: we only have %d kB across all osds!" % pgm['osd_stats_sum']['kb'])
if pgm['osd_stats_sum']['kb_used'] < 5 * 1024:
raise Exception("Refusing to reweight: we only have %d kB across all osds!" % pgm['osd_stats_sum']['kb_used'])
average_util = float(pgm['osd_stats_sum']['kb_used']) / float(pgm['osd_stats_sum']['kb'])
# adjust down only if we are above the threshold
overload_util = average_util * options.oload / 100.0
# adjust weights up whenever possible
underload_util = average_util - (overload_util - average_util)
print "average_util: %04f, overload_util: %04f, underload_util: %04f. " %(average_util, overload_util, underload_util)
print "reweighted: "
n = 0
if options.by_pg:
osds = sorted(pgm['osd_stats'], key=lambda osd: -abs(average_util - pgs_by_osd[osd['osd']] / get_weight(osd['osd'],type='crush_weight')))
else:
osds = sorted(pgm['osd_stats'], key=lambda osd: -abs(average_util - float(osd['kb_used']) / float(osd['kb'])))
for osd in osds:
if options.by_pg:
util = pgs_by_osd[osd['osd']] / get_weight(osd['osd'],type='crush_weight')
else:
util = float(osd['kb_used']) / float(osd['kb'])
# skip the OSDs that do not contain anything (probably a different crush root)
if util < 0.01:
continue
if util >= overload_util:
# Assign a lower weight to overloaded OSDs. The current weight
# is a factor to take into account the original weights,
# to represent e.g. differing storage capacities
weight = get_weight(osd['osd'])
new_weight = (average_util / util) * float(weight)
if weight - new_weight > options.max_change:
new_weight = weight - options.max_change
print "%d (%4f >= %4f) [%04f -> %04f]" % (osd['osd'], util, overload_util, weight, new_weight)
if options.doit: change_weight(osd['osd'], new_weight, options.really)
n += 1
if n >= options.num_osds: break
if options.adjust_up and util <= underload_util:
# assign a higher weight.. if we can
weight = get_weight(osd['osd'])
new_weight = (average_util / util) * float(weight)
if new_weight - weight > options.max_change:
new_weight = weight + options.max_change
if new_weight > 1.0:
new_weight = 1.0
if new_weight > weight:
print "%d (%4f <= %4f) [%04f -> %04f]" % (osd['osd'], util, underload_util, weight, new_weight)
if options.doit: change_weight(osd['osd'], new_weight, options.really)
n += 1
if n >= options.num_osds: break
#!/usr/bin/env python
from cephinfo import cephinfo
import itertools
import random
# read the ceph PG and OSD info from the ceph-mon
cephinfo.init_pg()
cephinfo.init_osd()
osds = [ osd['osd'] for osd in cephinfo.get_osds_data() ]
triple_combinations = [ tuple(pg['acting']) for pg in cephinfo.get_pg_stats() ]
print "We have %d OSDs and %d PGs, hence %d combinations e.g. like this: %s" % (len(osds), cephinfo.get_n_pgs(), len(triple_combinations), triple_combinations[0])
nFailures = 0
nDataLoss = 0
nSimulations = 1000
print "Simulating %d triple failures" % nSimulations
for i in xrange(0,nSimulations):
triple_failure = random.sample(osds, 3)
nFailures += 1
for order in itertools.permutations(triple_failure):
if order in triple_combinations:
nDataLoss += 1
print "3 replica data loss incident with failure %s" % str(order)
def reweight_by_utilization(options):
if options.oload <= 100:
raise Exception("You must give a percentage higher than 100.")
# "The reweighting threshold will be calculated as <average-utilization> "
# "times <input-percentage>. For example, an argument of 200 would "
# "reweight OSDs which are twice as utilized as the average OSD.\n";
cephinfo.init_pg()
pgm = cephinfo.pg_data
pgs_by_osd = defaultdict(int)
if options.by_pg:
weight_sum = 0.0
num_pg_copies = 0
num_osds = 0
for p in pgm['pg_stats']:
pool = p['pgid'].split('.')[0]
if options.pools and pool not in options.pools:
continue
for q in p['up']:
if not pgs_by_osd[q]:
pgs_by_osd[q] = 0
weight_sum += get_weight(q, 'crush_weight')
num_osds += 1
pgs_by_osd[q] += 1
num_pg_copies += 1
if not num_osds or (num_pg_copies / num_osds <
mon_reweight_min_pgs_per_osd):
raise Exception(
'Refusing to reweight: we only have %d PGs across %d osds!' %
(num_pg_copies, num_osds))
average_util = num_pg_copies / weight_sum
print "weight_sum: %3f, num_pg_copies: %d, num_osds: %d" % (
weight_sum, num_pg_copies, num_osds)
else:
num_osd = len(pgm['osd_stats'])
# Avoid putting a small number (or 0) in the denominator when calculating
# average_util
if pgm['osd_stats_sum'][
'kb'] * 1024 / num_osd < mon_reweight_min_bytes_per_osd:
raise Exception(
"Refusing to reweight: we only have %d kB across all osds!" %
pgm['osd_stats_sum']['kb'])
if pgm['osd_stats_sum']['kb_used'] < 5 * 1024:
raise Exception(
"Refusing to reweight: we only have %d kB across all osds!" %
pgm['osd_stats_sum']['kb_used'])
average_util = float(pgm['osd_stats_sum']['kb_used']) / float(
pgm['osd_stats_sum']['kb'])
# adjust down only if we are above the threshold
overload_util = average_util * options.oload / 100.0
# but aggressively adjust weights up whenever possible
underload_util = average_util # - (overload_util - average_util)
print "average_util: %04f, overload_util: %04f, underload_util: %04f. " % (
average_util, overload_util, underload_util)
print "reweighted: "
# sort to get heaviest osds first
nonempty_osds = [
osd for osd in pgm['osd_stats'] if float(osd['kb']) > 0
and get_weight(osd['osd'], type='crush_weight') > 0
]
if options.by_pg:
osds = sorted(
nonempty_osds,
key=lambda osd: -abs(average_util - pgs_by_osd[osd['osd']] /
get_weight(osd['osd'], type='crush_weight')))
else:
osds = sorted(nonempty_osds,
key=lambda osd: -abs(average_util - float(osd['kb_used'])
/ float(osd['kb'])))
n = 0
for osd in osds:
if options.by_pg:
util = pgs_by_osd[osd['osd']] / get_weight(osd['osd'],
type='crush_weight')
else:
util = float(osd['kb_used']) / float(osd['kb'])
# skip the OSDs that do not contain anything (probably a different crush root)
if util < 0.01:
continue
if util >= overload_util:
# Assign a lower weight to overloaded OSDs. The current weight
# is a factor to take into account the original weights,
# to represent e.g. differing storage capacities
weight = get_weight(osd['osd'])
new_weight = (average_util / util) * float(weight)
new_weight = max(new_weight, weight - options.max_change)
print "%d (%4f >= %4f) [%04f -> %04f]" % (
osd['osd'], util, overload_util, weight, new_weight)
if options.doit:
change_weight(osd['osd'], new_weight, options.really)
n += 1
if n >= options.num_osds: break
if options.adjust_up and util <= underload_util:
# assign a higher weight.. if we can
weight = get_weight(osd['osd'])
new_weight = (average_util / util) * float(weight)
new_weight = min(new_weight, weight + options.max_change)
if new_weight > 1.0:
new_weight = 1.0
if new_weight > weight:
print "%d (%4f <= %4f) [%04f -> %04f]" % (
osd['osd'], util, underload_util, weight, new_weight)
if options.doit:
change_weight(osd['osd'], new_weight, options.really)
n += 1
if n >= options.num_osds: break
def reweight_by_utilization(options):
if options.oload <= 100:
raise Exception("You must give a percentage higher than 100.")
cephinfo.init_pg()
pgm = cephinfo.pg_data
pgs_by_osd = defaultdict(int)
if options.by_pg:
weight_sum = 0.0
num_pg_copies = 0
num_osds = 0
for p in pgm['pg_stats']:
pool = p['pgid'].split('.')[0]
if options.pools and pool not in options.pools:
continue
for q in p['up']:
if not pgs_by_osd[q]:
pgs_by_osd[q] = 0
weight_sum += get_weight(q,'crush_weight')
num_osds += 1
pgs_by_osd[q] += 1
num_pg_copies += 1
if not num_osds or (num_pg_copies / num_osds < mon_reweight_min_pgs_per_osd):
raise Exception('Refusing to reweight: we only have %d PGs across %d osds!' % (num_pg_copies, num_osds))
average_util = num_pg_copies / weight_sum
if VERBOSE: print "weight_sum: %3f, num_pg_copies: %d, num_osds: %d" % (weight_sum, num_pg_copies, num_osds)
else:
num_osd = len(pgm['osd_stats'])
if num_osd < MIN_OSDS:
raise Exception("Refusing to reweight: we have only %d OSDs! (%d needed)" % (num_osd, MIN_OSDS))
# Avoid putting a small number (or 0) in the denominator when calculating average_util
if pgm['osd_stats_sum']['kb'] * 1024 / num_osd < mon_reweight_min_bytes_per_osd:
raise Exception("Refusing to reweight: we only have %d GB total space across all osds! (%d GB needed)" % (pgm['osd_stats_sum']['kb'] / 1024 / 1024, mon_reweight_min_bytes_per_osd * num_osd / 1024 / 1024 / 1024))
if pgm['osd_stats_sum']['kb_used'] * 1024 / num_osd < mon_reweight_min_bytes_used_per_osd:
raise Exception("Refusing to reweight: we only have %d GB used across all osds! (%d GB needed)" % (pgm['osd_stats_sum']['kb_used'] / 1024 / 1024, mon_reweight_min_bytes_used_per_osd * num_osd / 1024 / 1024 / 1024))
average_util = float(pgm['osd_stats_sum']['kb_used']) / float(pgm['osd_stats_sum']['kb'])
if VERBOSE: print "Found %d OSDs in total" % len(pgm['osd_stats'])
# filter out the empty osds
nonempty_osds = [ osd for osd in pgm['osd_stats'] if float(osd['kb']) > 0 and get_weight(osd['osd'],type='crush_weight') > 0 ]
if VERBOSE: print "Found %d non-empty OSDs" % len(nonempty_osds)
# optionally filter out osds not in the requested bucket
# and recalculate average_util
if options.bucket:
bucket_osds = []
for bucket in options.bucket:
bucket_osds.extend(ceph_osds_in_bucket.list(bucket))
sum_kb = 0
sum_weight = 0
sum_kb_used = 0
filtered_osds = []
for osd in nonempty_osds:
name = 'osd.%d' % osd['osd']
if name in bucket_osds:
sum_weight += get_weight(osd['osd'], 'crush_weight') * 1024*1024*1024
sum_kb_used += osd['kb_used']
filtered_osds.insert(0, osd)
average_util = float(sum_kb_used) / float(sum_weight)
if VERBOSE: print "Found %d OSDs after filtering by bucket" % len(filtered_osds)
else:
filtered_osds = nonempty_osds
# sort osds from most to least deviant from the average_util
if options.by_pg:
osds = sorted(filtered_osds, key=lambda osd: -abs(average_util - pgs_by_osd[osd['osd']] / get_weight(osd['osd'],type='crush_weight')))
else:
#osds = sorted(filtered_osds, key=lambda osd: -abs(average_util - float(osd['kb_used']) / float(osd['kb'])))
osds = sorted(filtered_osds, key=lambda osd: -abs(average_util - float(osd['kb_used']) / (get_weight(osd['osd'],type='crush_weight') * 1024*1024*1024)))
# adjust down only if we are above the threshold
overload_util = average_util * options.oload / 100.0
# but aggressively adjust weights up whenever possible
underload_util = average_util
if VERBOSE: print "average_util: %04f, overload_util: %04f, underload_util: %04f. " %(average_util, overload_util, underload_util)
n = 0
for osd in osds:
if options.by_pg:
util = pgs_by_osd[osd['osd']] / get_weight(osd['osd'],type='crush_weight')
else:
util = float(osd['kb_used']) / (get_weight(osd['osd'],type='crush_weight') * 1024*1024*1024)
# skip very empty OSDs
if util < 0.001:
continue
if util >= overload_util:
# Assign a lower weight to overloaded OSDs. The current weight
# is a factor to take into account the original weights,
# to represent e.g. differing storage capacities
weight = get_weight(osd['osd'])
new_weight = (average_util / util) * float(weight)
new_weight = max(new_weight, weight - options.max_change)
print "osd.%d (%4f >= %4f) [%04f -> %04f]" % (osd['osd'], util, overload_util, weight, new_weight)
if options.doit: change_weight(osd['osd'], new_weight, options.really)
n += 1
if n >= options.num_osds: break
if not options.no_increasing and util <= underload_util:
# assign a higher weight.. if we can
weight = get_weight(osd['osd'])
new_weight = (average_util / util) * float(weight)
new_weight = max(new_weight, weight + 0.01)
new_weight = min(new_weight, weight + options.max_change)
if new_weight > 1.0:
new_weight = 1.0
if new_weight > weight:
print "osd.%d (%4f <= %4f) [%04f -> %04f]" % (osd['osd'], util, underload_util, weight, new_weight)
if options.doit: change_weight(osd['osd'], new_weight, options.really)
n += 1
if n >= options.num_osds: break
def reweight_by_utilization(oload, by_pg, pools, doit, really):
if oload <= 100:
raise Exception("You must give a percentage higher than 100.")
# "The reweighting threshold will be calculated as <average-utilization> "
# "times <input-percentage>. For example, an argument of 200 would "
# "reweight OSDs which are twice as utilized as the average OSD.\n";
cephinfo.init_pg()
pgm = cephinfo.pg_data
pgs_by_osd = defaultdict(int)
if by_pg:
weight_sum = 0.0
num_pg_copies = 0
num_osds = 0
for p in pgm['pg_stats']:
pool = p['pgid'].split('.')[0]
if pools and pool not in pools:
continue
for a in p['acting']:
if not pgs_by_osd[a]:
num_osds += 1
weight_sum += get_weight(a, 'crush_weight')
pgs_by_osd[a] = 0
pgs_by_osd[a] += 1
num_pg_copies += 1
if not num_osds or (num_pg_copies / num_osds < 10):
raise Exception(
'Refusing to reweight: we only have %d PGs across %d osds!' %
(num_pg_copies, num_osds))
average_util = num_pg_copies / weight_sum
print "weight_sum: %3f, num_pg_copies: %d, num_osds: %d" % (
weight_sum, num_pg_copies, num_osds)
else:
# Avoid putting a small number (or 0) in the denominator when calculating
# average_util
if pgm['osd_stats_sum']['kb'] < 1024:
raise Exception(
"Refusing to reweight: we only have %d kB across all osds!" %
pgm['osd_stats_sum']['kb'])
if pgm['osd_stats_sum']['kb_used'] < 5 * 1024:
raise Exception(
"Refusing to reweight: we only have %d kB across all osds!" %
pgm['osd_stats_sum']['kb_used'])
average_util = float(pgm['osd_stats_sum']['kb_used']) / float(
pgm['osd_stats_sum']['kb'])
# adjust down only if we are above the threshold
overload_util = average_util * oload / 100.0
# adjust weights up whenever possible
underload_util = average_util # - (overload_util - average_util)
print "average_util: %04f, overload_util: %04f, underload_util: %04f. " % (
average_util, overload_util, underload_util)
print "reweighted: "
for osd in pgm['osd_stats']:
if by_pg:
util = pgs_by_osd[osd['osd']] / get_weight(osd['osd'],
type='crush_weight')
else:
util = float(osd['kb_used']) / float(osd['kb'])
# skip the OSDs that do not contain anything (probably a different crush root)
if util < 0.01:
continue
if util >= overload_util:
# Assign a lower weight to overloaded OSDs. The current weight
# is a factor to take into account the original weights,
# to represent e.g. differing storage capacities
weight = get_weight(osd['osd'])
new_weight = (average_util / util) * float(weight)
print "%d (%4f >= %4f) [%04f -> %04f]" % (
osd['osd'], util, overload_util, weight, new_weight)
if doit: change_weight(osd['osd'], new_weight, really)
if util <= underload_util:
# assign a higher weight.. if we can
weight = get_weight(osd['osd'])
new_weight = (average_util / util) * float(weight)
if new_weight > 1.0:
new_weight = 1.0
if new_weight > weight:
print "%d (%4f <= %4f) [%04f -> %04f]" % (
osd['osd'], util, underload_util, weight, new_weight)
if doit: change_weight(osd['osd'], new_weight, really)
#!/usr/bin/env python
from cephinfo import cephinfo
import itertools
import random
# read the ceph PG and OSD info from the ceph-mon
cephinfo.init_pg()
cephinfo.init_osd()
osds = [osd['osd'] for osd in cephinfo.get_osds_data()]
triple_combinations = [tuple(pg['acting']) for pg in cephinfo.get_pg_stats()]
print "We have %d OSDs and %d PGs, hence %d combinations e.g. like this: %s" % (
len(osds), cephinfo.get_n_pgs(), len(triple_combinations),
triple_combinations[0])
nFailures = 0
nDataLoss = 0
nSimulations = 1000
print "Simulating %d triple failures" % nSimulations
for i in xrange(0, nSimulations):
triple_failure = random.sample(osds, 3)
nFailures += 1
for order in itertools.permutations(triple_failure):
if order in triple_combinations:
nDataLoss += 1
