def test_compare_bucket_indep(self):
origin = self.define_crushmap_10()
# indep, mapping order does not matter
c = Main().constructor([
'compare',
'--rule', 'indep',
'--replication-count', '2',
'--order-matters',
])
c.set_origin_crushmap(origin)
#
# Swapping weights within a bucket, values stay
# in the bucket but move between children
#
destination = copy.deepcopy(origin)
host0 = destination['trees'][0]['children'][0]
w0 = host0['children'][0]['weight']
w1 = host0['children'][1]['weight']
host0['children'][0]['weight'] = w1
host0['children'][1]['weight'] = w0
c.set_destination_crushmap(destination)
c.args.values_count = 100
(from_to, in_out) = c.compare_bucket(host0)
assert from_to == {
'device01': {'device00': 5}
}
assert in_out == {}
#
# Increasing the weight of the items it contains changes the
# weight of the bucket and items move in/out of the bucket.
#
destination = copy.deepcopy(origin)
host0 = destination['trees'][0]['children'][0]
host0['children'][0]['weight'] *= 10
c.set_destination_crushmap(destination)
c.args.values_count = 30
(from_to, in_out) = c.compare_bucket(host0)
print("from_to " + str(from_to))
print("in_out " + str(in_out))
assert from_to == {
'device01': {'device00': 1}
}
assert in_out == {
'device01': {'device05': 1},
'device04': {'device00': 1},
'device05': {'device00': 2},
'device09': {'device01': 1},
'device13': {'device00': 1},
'device17': {'device00': 1},
'device19': {'device00': 1},
'device00': {'device19': 1},
}
def test_destination_weights(self):
a = Main().constructor([
"compare", "--rule", "replicated_ruleset",
"--replication-count", "1",
"--origin", "tests/weights-crushmap.json",
"--destination", "tests/weights-crushmap.json",
"--destination-weights", "tests/weights.json"])
a.args.backward_compatibility = True
a.run_compare()
def test_analyze_weights(self):
a = Main().constructor([
"analyze", "--rule", "replicated_ruleset", "--replication-count",
"2", "--type", "device", "--crushmap", "tests/ceph/dump.json",
"--weights", "tests/ceph/weights.json"
])
a.args.backward_compatibility = True
res = a.run()
assert "-100.00" in str(res) # One of the OSDs has a weight of 0.0
def verify_optimize(self, weights, expected_delta, values_count,
replication_count):
crushmap = self.make_bucket(weights)
rule = 'firstn'
p = [
'--values-count',
str(values_count),
'--replication-count',
str(replication_count),
'--rule',
rule,
'--choose-args',
'optimize',
]
o = Main().constructor(['--verbose', 'optimize'] + p)
origin_crushmap = copy.deepcopy(crushmap)
bucket = crushmap['trees'][0]
previous_weight_set = []
for position in range(replication_count):
o.optimize_replica(p, origin_crushmap, crushmap, bucket,
replication_count, position)
assert 1 == len(crushmap['choose_args']['optimize'])
weight_set = copy.deepcopy(
crushmap['choose_args']['optimize'][0]['weight_set'])
assert weight_set[:len(previous_weight_set)] == previous_weight_set
a = Main().constructor(['analyze', '--choose-args', 'optimize'] + p)
r = a.analyze_crushmap(crushmap)
delta = (r['~objects~'] - r['~expected~']).tolist()
print("delta = " + str(delta))
assert expected_delta == delta
def test_display(self):
c1, c2 = self.define_crushmaps_1()
c = Main().constructor([
'compare',
'--rule', 'indep',
'--replication-count', '2',
'--values-count', '10',
])
c.set_origin(c1)
c.set_destination(c2)
c.compare()
out = c.display()
print(out)
assert 'device04 0 0 1 1 10.00%' in out
assert 'objects% 10.00% 5.00% 5.00% 5.00% 25.00%' in out
def test_compare_display(self):
c = Main().constructor([
'compare',
'--rule', 'firstn',
'--values-count', '1000',
])
c1, c2 = self.define_crushmaps_2()
c.set_origin(c2)
c.set_destination(c1)
c.args.replication_count = 1
c.compare()
out = c.display()
print(out)
assert "objects% 10.20% 12.70% 22.90%" in out
c.args.replication_count = 3
c.compare()
out = c.display()
print(out)
assert ("objects% 0.17% 0.13% 0.97% 0.77% 0.77%"
" 0.63% 0.73% 0.87% 12.30% 12.20% 29.53%") in out
def test_sanity_check_args(self):
a = Main().constructor([
'compare',
])
with pytest.raises(Exception) as e:
a.pre_sanity_check_args()
assert 'missing --origin' in str(e.value)
a = Main().constructor([
'compare',
'--origin', 'ORIGIN',
])
with pytest.raises(Exception) as e:
a.pre_sanity_check_args()
assert 'missing --destination' in str(e.value)
a = Main().constructor([
'compare',
'--origin', 'ORIGIN',
'--destination', 'DESTINATION',
])
a.pre_sanity_check_args()
with pytest.raises(Exception) as e:
a.post_sanity_check_args()
assert 'missing --rule' in str(e.value)
a = Main().constructor([
'compare',
'--origin', 'ORIGIN',
'--destination', 'DESTINATION',
'--rule', 'RULE',
'--choose-args', 'CHOOSE ARGS',
])
a.pre_sanity_check_args()
a.post_sanity_check_args()
def test_compare(self):
c1, c2 = self.define_crushmaps_1()
# firstn, mapping order does not matter
c = Main().constructor([
'compare',
'--rule', 'firstn',
'--replication-count', '2',
'--values-count', '10',
])
# device05 is removed
c.set_origin(c1)
c.set_destination(c2)
assert c.compare() == {
'device04': {'device17': 1},
'device05': {'device12': 1, 'device04': 2, 'device08': 1}
}
# device05 is added
c.set_origin(c2)
c.set_destination(c1)
assert c.compare() == {
'device12': {'device05': 1},
'device17': {'device04': 1},
'device04': {'device05': 2},
'device08': {'device05': 1}
}
# indep, mapping order matters
c = Main().constructor([
'compare',
'--rule', 'indep',
'--replication-count', '2',
'--values-count', '10',
'--order-matters',
])
# device05 is removed
c.set_origin(c1)
c.set_destination(c2)
assert c.compare() == {
'device04': {'device13': 1, 'device17': 1},
'device05': {'device04': 2, 'device08': 1}
}
# device05 is added
c.set_origin(c2)
c.set_destination(c1)
assert c.compare() == {
'device13': {'device04': 1},
'device17': {'device04': 1},
'device04': {'device05': 2},
'device08': {'device05': 1}
}
def test_sanity_check_args(self):
a = Main().constructor([
'optimize',
])
with pytest.raises(Exception) as e:
a.pre_sanity_check_args()
assert 'missing --crushmap' in str(e.value)
a = Main().constructor([
'optimize',
'--crushmap',
'CRUSHMAP',
])
with pytest.raises(Exception) as e:
a.pre_sanity_check_args()
assert 'missing --out-path' in str(e.value)
a = Main().constructor([
'optimize',
'--crushmap',
'CRUSHMAP',
'--out-path',
'OUT PATH',
'--no-forecast',
])
with pytest.raises(Exception) as e:
a.pre_sanity_check_args()
assert 'only valid with --step' in str(e.value)
a = Main().constructor([
'optimize',
'--crushmap',
'CRUSHMAP',
'--out-path',
'OUT PATH',
])
a.pre_sanity_check_args()
with pytest.raises(Exception) as e:
a.post_sanity_check_args()
assert 'missing --rule' in str(e.value)
a = Main().constructor([
'optimize',
'--crushmap',
'CRUSHMAP',
'--out-path',
'OUT PATH',
'--rule',
'RULE',
])
a.pre_sanity_check_args()
with pytest.raises(Exception) as e:
a.post_sanity_check_args()
assert 'missing --choose-args' in str(e.value)
a = Main().constructor([
'optimize',
'--crushmap',
'CRUSHMAP',
'--out-path',
'OUT PATH',
'--rule',
'RULE',
'--choose-args',
'CHOOSE ARGS',
])
a.pre_sanity_check_args()
a.post_sanity_check_args()
def test_analyze(self):
trees = [
{
"name": "dc1",
"type": "root",
"id": -1,
'children': []
},
]
weights = (
(10.0, 1.0, 5.0, 4.0),
(10.0, 1.0, 5.0, 4.0),
(10.0, 1.0, 5.0, 4.0),
(10.0, 1.0, 5.0, 4.0),
(1.0, 0.1, 0.5, 0.4),
)
trees[0]['children'].extend([{
"type":
"host",
"id":
-(i + 3),
"name":
"host%d" % i,
"weight":
weights[i][0],
"children": [
{
"id": (3 * i),
"name": "device%02d" % (3 * i),
"weight": weights[i][1]
},
{
"id": (3 * i + 1),
"name": "device%02d" % (3 * i + 1),
"weight": weights[i][2]
},
{
"id": (3 * i + 2),
"name": "device%02d" % (3 * i + 2),
"weight": weights[i][3]
},
],
} for i in range(5)])
a = Main().constructor([
'analyze',
'--rule',
'data',
'--replication-count',
'2',
'--values-count',
'10000',
])
a.args.crushmap = {
"trees": trees,
"rules": {
"data": [["take", "dc1"],
["chooseleaf", "firstn", 0, "type", "host"], ["emit"]]
}
}
d = a.analyze()
expected = """\
~id~ ~weight~ ~objects~ ~over/under used %~
~name~
host4 -7 1.0 541 10.91
host3 -6 10.0 4930 1.07
host2 -5 10.0 4860 -0.37
host1 -4 10.0 4836 -0.86
host0 -3 10.0 4833 -0.92
Worst case scenario if a host fails:
~over used %~
~type~
device 25.55
host 22.45
root 0.00\
""" # noqa trailing whitespaces are expected
assert expected == str(d)