def test_report_compat_hammer(self):
#
# hammer does not know about the stable tunable, verify this
# is handled properly. It must be set to zero otherwise the
# ceph report below will have unexpected mappings.
#
a = Ceph().constructor([
'--verbose',
'analyze',
'--crushmap', 'tests/ceph/ceph-report-compat-hammer.json',
'--pool', '42',
])
d = a.analyze_report(*a.analyze())
print(str(d))
expected = """\
~id~ ~weight~ ~PGs~ ~over/under filled %~
~name~
node-8v -6 1.08 5 56.25
node-5v -3 1.08 4 25.00
node-6v -4 1.08 3 -6.25
node-7v -5 1.08 3 -6.25
node-4 -2 1.08 1 -68.75
Worst case scenario if a host fails:
~over filled %~
~type~
device 150.0
host 75.0
root 0.0\
""" # noqa trailing whitespaces are expected
assert expected == str(d)
def test_report_compat(self):
#
# verify --choose-args is set to the pool when the crushmap contains
# *-target-weights buckets.
#
for p in ([],
['--choose-args=3'],
['--pool=3'],
['--choose-args=3', '--pool=3']):
a = Ceph().constructor([
'--verbose',
'analyze',
'--crushmap', 'tests/ceph/ceph-report-compat.json',
] + p)
d = a.analyze_report(*a.analyze())
print(str(d))
expected = """\
~id~ ~weight~ ~PGs~ ~over/under filled %~
~name~
host0 -1 1.0 1 0.0
host1 -2 1.0 1 0.0
host2 -5 1.0 1 0.0
Worst case scenario if a host fails:
~over filled %~
~type~
device 33.33
host 33.33
rack 0.00
root 0.00\
""" # noqa trailing whitespaces are expected
assert expected == str(d)
def test_out_version(self):
expected_path = 'tests/sample-ceph-crushmap-compat.txt'
out_path = expected_path + ".err"
in_path = 'tests/sample-ceph-crushmap-compat.python-json'
Ceph().main([
'convert',
'--in-path',
in_path,
'--out-path',
out_path,
'--out-format',
'txt',
'--out-version',
'jewel',
])
assert os.system("diff -Bbu " + expected_path + " " + out_path) == 0
os.unlink(out_path)
in_path = 'tests/sample-ceph-crushmap.python-json'
with pytest.raises(Exception) as e:
Ceph().main([
'convert',
'--in-path',
in_path,
'--out-path',
out_path,
'--out-format',
'txt',
'--out-version',
'jewel',
])
assert 'version lower than luminous' in str(e.value)
def test_optimize_one_step(self):
pg_num = 2048
size = 3
a = Ceph().constructor([
'optimize',
'--no-multithread',
'--replication-count',
str(size),
'--pool',
'3',
'--pg-num',
str(pg_num),
'--pgp-num',
str(pg_num),
'--rule',
'data',
'--choose-args',
'optimize',
'--step',
'64',
])
c = Crush(backward_compatibility=True)
c.parse('tests/test_optimize_small_cluster.json')
crushmap = c.get_crushmap()
(count, crushmap) = a.optimize(crushmap)
assert 240 == count
def test_get_choose_arg(self):
a = Ceph().constructor([
'optimize',
'--choose-args',
'optimize',
])
crushmap = {}
bucket = {'id': -1}
choose_arg = a.get_choose_arg(crushmap, bucket)
assert {'bucket_id': -1} == choose_arg
assert [choose_arg] == crushmap['choose_args']['optimize']
choose_arg = a.get_choose_arg(crushmap, bucket)
assert {'bucket_id': -1} == choose_arg
assert [choose_arg] == crushmap['choose_args']['optimize']
bucket = {'id': -2}
choose_arg = a.get_choose_arg(crushmap, bucket)
assert [{
'bucket_id': -1
}, {
'bucket_id': -2
}] == crushmap['choose_args']['optimize']
def test_optimize_step(self):
pg_num = 2048
size = 3
a = Ceph().constructor([
'optimize',
'--no-multithread',
'--replication-count',
str(size),
'--pool',
'3',
'--pg-num',
str(pg_num),
'--pgp-num',
str(pg_num),
'--rule',
'data',
'--choose-args',
'optimize',
'--step',
'64',
])
c = Crush(backward_compatibility=True)
c.parse('tests/test_optimize_small_cluster.json')
crushmap = c.get_crushmap()
converged = False
for i in range(20):
(count, crushmap) = a.optimize(crushmap)
if count <= 0:
converged = True
break
print("moved " + str(count) + " values")
assert converged
def test_hook_create_values(self):
c = Ceph()
c.parse([
'--verbose',
'analyze',
'--values-count',
'2',
])
assert {0: 0, 1: 1} == c.hook_create_values()
c.parse([
'--verbose',
'analyze',
'--pool',
'2',
'--pg-num',
'3',
'--pgp-num',
'3',
])
expected = {
u'2.0': -113899774,
u'2.1': -1215435108,
u'2.2': -832918304
}
assert expected == c.hook_create_values()
def test_optimize_report_compat_two_pools(self):
expected_path = 'tests/ceph/ceph-report-compat-optimized.txt'
out_path = expected_path + ".err"
for p in (['--pool=3'], ['--choose-args=3', '--pool=3']):
Ceph().main([
'--verbose',
'optimize',
'--no-multithread',
'--crushmap',
'tests/ceph/ceph-report-compat.json',
'--out-path',
out_path,
'--out-format',
'txt',
] + p)
assert os.system("diff -Bbu " + expected_path + " " +
out_path) == 0
os.unlink(out_path)
with pytest.raises(Exception) as e:
Ceph().main([
'--verbose',
'optimize',
'--no-multithread',
'--crushmap',
'tests/ceph/ceph-report-compat-two-pools.json',
'--out-path',
out_path,
'--out-format',
'txt',
])
assert '--pool is required' in str(e.value)
with pytest.raises(Exception) as e:
Ceph().main([
'--verbose',
'optimize',
'--no-multithread',
'--crushmap',
'tests/ceph/ceph-report-compat-two-pools.json',
'--out-path',
out_path,
'--out-format',
'txt',
'--pool',
'1324',
])
assert '1324 is not a known pool' in str(e.value)
def test_set_choose_arg_position(self):
a = Ceph().constructor([
'optimize',
'--choose-args',
'optimize',
])
bucket = {
'id': -1,
'children': [
{
'id': 1,
'weight': 10
},
{
'id': 2,
'weight': 20
},
],
}
choose_arg = {'bucket_id': -1}
a.set_choose_arg_position(choose_arg, bucket, 0)
assert {'bucket_id': -1, 'weight_set': [[10, 20]]} == choose_arg
a.set_choose_arg_position(choose_arg, bucket, 1)
assert {
'bucket_id': -1,
'weight_set': [[10, 20], [10, 20]]
} == choose_arg
choose_arg['weight_set'][1] = [100, 200]
a.set_choose_arg_position(choose_arg, bucket, 3)
expected = {
'bucket_id': -1,
'weight_set': [[10, 20], [100, 200], [100, 200], [100, 200]]
}
assert expected == choose_arg
a.set_choose_arg_position(choose_arg, bucket, 1)
assert expected == choose_arg
def test_report(self):
in_path = 'tests/ceph/ceph-report.json'
expected_path = 'tests/ceph/crushmap-from-ceph-report.json'
out_path = expected_path + ".err"
Ceph().main([
'convert',
'--in-path',
in_path,
'--out-path',
out_path,
'--out-format',
'python-json',
])
assert os.system("diff -Bbu " + expected_path + " " + out_path) == 0
os.unlink(out_path)
def test_rules_order(self):
expected_path = 'tests/ceph/ceph-crushmap-rules-order.txt'
out_path = expected_path + ".err"
Ceph().main([
'--verbose',
'convert',
'--in-path',
'tests/ceph/ceph-crushmap-rules-order.json',
'--out-path',
out_path,
'--out-format',
'txt',
])
assert os.system("diff -Bbu " + expected_path + " " + out_path) == 0
os.unlink(out_path)
def test_report_compat(self):
#
# verify --choose-args is set to the pool when the crushmap contains
# *-target-weights buckets.
#
expected_path = 'tests/ceph/ceph-report-compat-converted.txt'
out_path = expected_path + ".err"
Ceph().main([
'--verbose',
'convert',
'--in-path',
'tests/ceph/ceph-report-compat.json',
'--out-path',
out_path,
'--out-format',
'txt',
])
assert os.system("diff -Bbu " + expected_path + " " + out_path) == 0
os.unlink(out_path)
def test_optimize_step_forecast(self, caplog):
expected_path = 'tests/test_optimize_small_cluster_step_1.txt'
out_path = expected_path + ".err"
# few samples
pg_num = 2048
size = 3
Ceph().main([
'--verbose',
'optimize',
'--no-multithread',
'--crushmap',
'tests/test_optimize_small_cluster.json',
'--out-path',
out_path,
'--out-format',
'txt',
'--replication-count',
str(size),
'--pool',
'2',
'--pg-num',
str(pg_num),
'--pgp-num',
str(pg_num),
'--rule',
'data',
'--choose-args',
'0',
'--step',
'64',
])
assert os.system("diff -Bbu " + expected_path + " " + out_path) == 0
os.unlink(out_path)
assert 'step 2 moves 73 objects' in caplog.text()
assert 'step 3 moves 76 objects' in caplog.text()
assert 'step 4 moves 93 objects' in caplog.text()
assert 'step 5 moves 80 objects' in caplog.text()
assert 'step 6 moves 100 objects' in caplog.text()
assert 'step 7 moves 80 objects' in caplog.text()
assert 'step 8 moves 53 objects' in caplog.text()
assert 'step 9 moves 0 objects' in caplog.text()
def test_set_optimize_args(self):
a = Ceph().constructor([
'optimize',
'--pool',
'3',
])
a.args.replication_count = None
assert a.args.choose_args is None
assert a.args.rule is None
assert a.args.pg_num is None
assert a.args.pgp_num is None
assert a.args.out_version == 'luminous'
assert a.args.with_positions is True
a.main.convert_to_crushmap('tests/ceph/ceph-report-small.json')
assert a.args.replication_count == 3
assert a.args.choose_args == '3'
assert a.args.rule == 'data'
assert a.args.pg_num == 1
assert a.args.pgp_num == 1
assert a.args.out_version == 'j'
assert a.args.with_positions is False
def optimize(p, crushmap, bucket, with_positions):
if len(bucket.get('children', [])) == 0:
return None
print("Optimizing " + bucket['name'])
crushmap = copy.deepcopy(crushmap)
a = Ceph().constructor([
'analyze',
] + p)
id2weight = collections.OrderedDict([(i['id'], i['weight'])
for i in bucket['children']])
if with_positions:
choose_arg = {
'bucket_id': bucket['id'],
'weight_set': [
id2weight.values(),
] * a.args.replication_count
}
crushmap['choose_args']['optimize'].append(choose_arg)
for replication_count in range(1, a.args.replication_count + 1):
print("Improving replica " + str(replication_count))
pprint.pprint(choose_arg['weight_set'])
optimize_replica(a, crushmap, bucket, replication_count,
choose_arg, replication_count - 1)
else:
choose_arg = {
'bucket_id': bucket['id'],
'weight_set': [
id2weight.values(),
]
}
crushmap['choose_args']['optimize'].append(choose_arg)
optimize_replica(a, crushmap, bucket, a.args.replication_count,
choose_arg, 0)
print(bucket['name'] + " weights " + str(id2weight.values()))
pprint.pprint(choose_arg['weight_set'])
return choose_arg
def test_optimize_report_compat_one_pool(self):
#
# verify --choose-args is set to --pool when the crushmap contains
# *-target-weights buckets.
#
expected_path = 'tests/ceph/ceph-report-compat-optimized.txt'
out_path = expected_path + ".err"
for p in ([], ['--choose-args=3'], ['--pool=3'],
['--choose-args=3', '--pool=3']):
Ceph().main([
'--verbose',
'optimize',
'--no-multithread',
'--crushmap',
'tests/ceph/ceph-report-compat.json',
'--out-path',
out_path,
'--out-format',
'txt',
] + p)
assert os.system("diff -Bbu " + expected_path + " " +
out_path) == 0
os.unlink(out_path)
def test_conversions(self):
base = 'tests/sample-ceph-crushmap.'
for ext_in in ('txt', 'crush', 'json', 'python-json'):
in_path = base + ext_in
for ext_out in ('txt', 'crush', 'json', 'python-json'):
expected_path = base + ext_out
out_path = expected_path + ".err"
print("conversion " + in_path + " => " + expected_path)
Ceph().main([
'convert',
'--in-path',
in_path,
'--out-path',
out_path,
'--out-format',
ext_out,
])
if ext_out == 'crush':
cmd = "cmp"
else:
cmd = "diff -Bbu"
assert os.system(cmd + " " + expected_path + " " +
out_path) == 0
os.unlink(out_path)
def analyze_optimization(self, p, crushmap, optimized, gain):
a = Ceph().constructor(['analyze'] + p)
before = a.analyze_crushmap(crushmap)
print("============= before")
print(str(before))
a = Ceph().constructor(['analyze', '--choose-args', 'optimize'] + p)
after = a.analyze_crushmap(optimized)
print("============= after")
print(str(after))
for type in before['~type~'].unique():
b = before.loc[before['~type~'] == type]
b = b.sort_values(by='~over/under filled %~', ascending=False)
b_span = b.iloc[0]['~over/under filled %~'] - b.iloc[-1][
'~over/under filled %~']
a = after.loc[after['~type~'] == type]
a = a.sort_values(by='~over/under filled %~', ascending=False)
a_span = a.iloc[0]['~over/under filled %~'] - a.iloc[-1][
'~over/under filled %~']
print("============= span " + str(type) + " before " +
str(b_span) + " after " + str(a_span))
assert a_span <= b_span / gain
def test_analyze_out_of_bounds(self):
# [ 5 1 1 1 1]
size = 2
pg_num = 2048
p = [
'--replication-count',
str(size),
'--pool',
'0',
'--pg-num',
str(pg_num),
'--pgp-num',
str(pg_num),
]
hosts_count = 5
host_weight = [1] * hosts_count
host_weight[0] = 5
crushmap = {
"trees": [{
"type": "root",
"id": -1,
"name": "dc1",
"weight": sum(host_weight),
"children": [],
}],
"rules": {
"firstn": [["take", "dc1"],
["choose", "firstn", 0, "type", "host"], ["emit"]],
}
}
crushmap['trees'][0]['children'].extend([{
"type": "host",
"id": -(i + 2),
"name": "host%d" % i,
"weight": host_weight[i],
"children": [],
} for i in range(0, hosts_count)])
a = Ceph().constructor([
'analyze',
'--rule',
'firstn',
] + p)
a.args.crushmap = crushmap
d = a.analyze()
expected = """\
~id~ ~weight~ ~objects~ ~over/under used %~
~name~
host3 -5 1 646 41.94
host4 -6 1 610 34.03
host2 -4 1 575 26.34
host1 -3 1 571 25.46
host0 -2 5 1694 -25.56
Worst case scenario if a host fails:
~over used %~
~type~
host 61.52
root 0.00
The following are overweight:
~id~ ~weight~
~name~
host0 -2 5\
""" # noqa trailing whitespaces are expected
assert expected == str(d)
def test_sanity_check_args(self):
a = Ceph().constructor([
'optimize',
])
with pytest.raises(Exception) as e:
a.pre_sanity_check_args()
assert 'missing --crushmap' in str(e.value)
a = Ceph().constructor([
'optimize',
'--crushmap',
'CRUSHMAP',
'--out-path',
'OUT PATH',
'--rule',
'RULE',
'--choose-args',
'CHOOSE ARGS',
'--pool',
'3',
'--values-count',
'8',
])
a.pre_sanity_check_args()
with pytest.raises(Exception) as e:
a.post_sanity_check_args()
assert '--pool and --values-count are mutually exclusive' in str(
e.value)
a = Ceph().constructor([
'optimize',
'--crushmap',
'CRUSHMAP',
'--out-path',
'OUT PATH',
'--rule',
'RULE',
'--choose-args',
'CHOOSE ARGS',
])
a.pre_sanity_check_args()
a.post_sanity_check_args()
def test_pickle(self):
o = Ceph().constructor(['optimize'])
p = pickle.dumps(o)
oo = pickle.loads(p)
assert oo.main.argv == o.main.argv
assert type(oo) == type(o)
def run_optimize(self, p, crushmap, gain):
o = Ceph().constructor(['optimize', '--choose-args', 'optimize'] + p)
origin_crushmap = copy.deepcopy(crushmap)
(count, optimized) = o.optimize(crushmap)
self.analyze_optimization(p, origin_crushmap, optimized, gain)
def test_sanity_check_args(self):
a = Ceph().constructor([
'analyze',
])
with pytest.raises(Exception) as e:
a.pre_sanity_check_args()
assert 'missing --crushmap' in str(e.value)
a = Ceph().constructor([
'analyze',
'--crushmap', 'CRUSHMAP',
])
a.pre_sanity_check_args()
a = Ceph().constructor([
'analyze',
'--crushmap', 'CRUSHMAP',
])
with pytest.raises(Exception) as e:
a.post_sanity_check_args()
assert 'missing --rule' in str(e.value)
a = Ceph().constructor([
'analyze',
'--crushmap', 'CRUSHMAP',
'--rule', 'RULE',
])
a.post_sanity_check_args()
a = Ceph().constructor([
'analyze',
'--crushmap', 'CRUSHMAP',
'--rule', 'RULE',
'--pool', '3',
'--values-count', '8',
])
with pytest.raises(Exception) as e:
a.post_sanity_check_args()
assert '--pool and --values-count are mutually exclusive' in str(e.value)
a = Ceph().constructor([
'analyze',
'--crushmap', 'CRUSHMAP',
'--rule', 'RULE',
'--pool', '3',
])
with pytest.raises(Exception) as e:
a.post_sanity_check_args()
assert '--pg-num is required' in str(e.value)
a = Ceph().constructor([
'analyze',
'--crushmap', 'CRUSHMAP',
'--rule', 'RULE',
'--pool', '3',
'--pg-num', '10',
])
with pytest.raises(Exception) as e:
a.post_sanity_check_args()
assert '--pgp-num is required' in str(e.value)
a = Ceph().constructor([
'analyze',
'--crushmap', 'CRUSHMAP',
'--rule', 'RULE',
'--pool', '3',
'--pg-num', '10',
'--pgp-num', '10',
])
a.post_sanity_check_args()
def run_optimize(self, p, rule_name, crushmap, with_positions=True):
pd.set_option('display.max_rows', None)
pd.set_option('display.width', 160)
p.extend(['--rule', rule_name])
a = Ceph().constructor([
'analyze',
] + p)
c = Crush(backward_compatibility=True)
c.parse(crushmap)
(take, failure_domain) = c.rule_get_take_failure_domain(rule_name)
crushmap = c.get_crushmap()
crushmap['choose_args'] = {
"optimize": [],
}
d = a.run_simulation(c, take, failure_domain)
if d['~overweight~'].any():
raise ValueError(
'no way to optimize when there is an overweight item')
print(str(d))
print(a._format_report(d, 'device'))
print(a._format_report(d, failure_domain))
print(a.analyze_failures(c, take, failure_domain))
p.extend(['--choose-args', 'optimize'])
pool = Pool()
children = [c.find_bucket(take)]
while len(children) > 0:
a = [(p, crushmap, item, with_positions) for item in children]
r = pool.map(o, a)
# r = map(o, a)
choose_args = filter(None, r)
crushmap['choose_args']['optimize'].extend(choose_args)
nc = []
for item in children:
nc.extend(item.get('children', []))
# fail if all children are not of the same type
children = nc
pprint.pprint(crushmap)
c.parse(crushmap)
a = Ceph().constructor([
'analyze',
] + p)
d = a.run_simulation(c, take, failure_domain)
print(a._format_report(d, 'device'))
print(a._format_report(d, failure_domain))
print(a.analyze_failures(c, take, failure_domain))
def test_sanity_check_args(self):
a = Ceph().constructor([
'convert',
])
with pytest.raises(Exception) as e:
a.pre_sanity_check_args()
assert 'missing --in-path' in str(e.value)
a = Ceph().constructor([
'convert',
'--in-path',
'IN',
])
with pytest.raises(Exception) as e:
a.post_sanity_check_args()
assert 'missing --out-path' in str(e.value)
a = Ceph().constructor([
'convert',
'--in-path',
'IN',
'--out-path',
'OUT',
])
a.pre_sanity_check_args()
a.post_sanity_check_args()
def test_sanity_check_args(self):
a = Ceph().constructor([
'compare',
])
with pytest.raises(Exception) as e:
a.pre_sanity_check_args()
assert 'missing --origin' in str(e.value)
a = Ceph().constructor([
'compare',
'--origin',
'ORIGIN',
'--destination',
'DESTINATION',
'--rule',
'RULE',
'--origin-choose-args',
'ORIGIN CHOOSE ARGS',
'--destination-choose-args',
'DESTINATION CHOOSE ARGS',
])
a.pre_sanity_check_args()
a.post_sanity_check_args()
a = Ceph().constructor([
'compare',
'--origin',
'ORIGIN',
'--destination',
'DESTINATION',
'--rule',
'RULE',
'--origin-choose-args',
'ORIGIN CHOOSE ARGS',
'--destination-choose-args',
'DESTINATION CHOOSE ARGS',
'--pool',
'3',
'--values-count',
'10',
])
a.pre_sanity_check_args()
with pytest.raises(Exception) as e:
a.post_sanity_check_args()
assert 'mutually exclusive' in str(e.value)