def request_weight_update(node, new_weight):
path_s = " ".join("{0}={1}".format(tp, name)
for tp, name in node.full_path)
cmd = "ceph osd crush set {name} {weight} {path}".format(name=node.name,
weight=new_weight,
path=path_s)
run_locally(cmd)
def calculate_remap_crush(new_crush_f, pg_dump_f=None, osd_map_name=None):
with tempfile.NamedTemporaryFile() as osd_map_fd:
if not osd_map_name:
run_locally("ceph osd getmap -o {0}".format(osd_map_fd.name))
else:
osd_map_name = osd_map_fd.name
with tempfile.NamedTemporaryFile() as osd_map_new_fd:
shutil.copy(osd_map_name, osd_map_new_fd.name)
run_locally("osdmaptool --import-crush {0} {1}".format(
new_crush_f, osd_map_new_fd.name))
return calculate_remap(osd_map_name,
osd_map_new_fd.name,
pg_dump_f=pg_dump_f)
def calculate_remap(curr_map_f, new_map_f, pg_dump_f=None):
curr_distr = run_locally(
"osdmaptool --test-map-pgs-dump {0}".format(curr_map_f)).decode("utf8")
curr_pools = {pool.pid: pool for pool in parse(curr_distr)}
new_distr = run_locally(
"osdmaptool --test-map-pgs-dump {0}".format(new_map_f)).decode("utf8")
new_pools = {pool.pid: pool for pool in parse(new_distr)}
pool_pairs = {
pool.pid: (curr_pools[pool.pid], pool)
for pool in new_pools.values()
}
pg_dump_js = open(pg_dump_f).read() if pg_dump_f else None
pg_sizes = get_pg_sizes(pg_dump_js)
return get_osd_diff(pool_pairs, pg_sizes)
def is_rebalance_complete(allowed_states=("active+clean", "active+remapped")):
pg_stat = json.loads(
run_locally("ceph pg stat --format=json").decode('utf8'))
if 'num_pg_by_state' in pg_stat:
for dct in pg_stat['num_pg_by_state']:
if dct['name'] not in allowed_states and dct["num"] != 0:
return False
else:
ceph_state = json.loads(
run_locally("ceph -s --format=json").decode('utf8'))
for pg_stat_dict in ceph_state['pgmap']['pgs_by_state']:
if pg_stat_dict[
'state_name'] not in allowed_states and pg_stat_dict[
'count'] != 0:
return False
return True
def get_pg_dump(pg_dump_js=None):
if pg_dump_js is None:
pg_dump_js = run_locally("ceph pg dump --format=json").decode("utf8")
if isinstance(pg_dump_js, str):
return json.loads(pg_dump_js)
else:
assert isinstance(pg_dump_js, dict)
return pg_dump_js
def get_creds_openrc(path: str) -> Tuple[str, str, str, str, bool]:
fc = open(path).read()
echo = 'echo "$OS_INSECURE:$OS_TENANT_NAME:$OS_USERNAME:$OS_PASSWORD@$OS_AUTH_URL"'
msg = "Failed to get creads from openrc file"
with LogError(msg):
data = run_locally(['/bin/bash'],
input_data=(fc + "\n" +
echo).encode('utf8')).decode("utf8")
msg = "Failed to get creads from openrc file: " + data
with LogError(msg):
data = data.strip()
insecure_str, user, tenant, passwd_auth_url = data.split(':', 3)
insecure = (insecure_str in ('1', 'True', 'true'))
passwd, auth_url = passwd_auth_url.rsplit("@", 1)
assert (auth_url.startswith("https://")
or auth_url.startswith("http://"))
return user, passwd, tenant, auth_url, insecure
def load_all_data(opts, no_cache=False):
if no_cache or not opts.osd_map:
if opts.offline:
logger.error(
"Can't get osd's map in offline mode as --osd-map is not passed from CLI"
)
return None, None, None, None
else:
osd_map_f = tmpnam()
run_locally("ceph osd getmap -o {0}".format(osd_map_f))
else:
osd_map_f = opts.osd_map
crushmap_bin_f = tmpnam()
run_locally("osdmaptool --export-crush {0} {1}".format(
crushmap_bin_f, osd_map_f))
crushmap_txt_f = tmpnam()
run_locally("crushtool -d {0} -o {1}".format(crushmap_bin_f,
crushmap_txt_f))
if no_cache or not opts.osd_tree:
if opts.offline:
logger.warning(
"Can't get osd's reweright in offline mode as --osd-tree is not passed from CLI"
)
osd_tree_js = None
else:
osd_tree_js = run_locally("ceph osd tree --format=json").decode(
"utf8")
else:
osd_tree_js = open(opts.osd_tree).read()
curr_reweight = {}
if osd_tree_js:
for node in json.loads(osd_tree_js)['nodes']:
if node['type'] == 'osd':
curr_reweight[node['name']] = node.get('reweight', 1.0)
crush = load_crushmap(crushmap_txt_f)
return crush, curr_reweight, crushmap_bin_f, osd_map_f
def main(argv):
opts = parse_args(argv[1:])
default_level = logging.DEBUG if opts.verbose else logging.WARNING
setup_loggers([clogger, logger], default_level=default_level)
if opts.subparser_name == 'dump':
crush_map_f = tmpnam()
if opts.osd_map:
run_locally("osdmaptool --export-crush {0} {1}".format(
crush_map_f, opts.osd_map))
else:
run_locally("ceph osd getcrushmap -o {0}".format(crush_map_f))
run_locally("crushtool -d {0} -o {1}".format(crush_map_f,
opts.out_file))
return 0
if opts.osd_map:
osd_map_f = opts.osd_map
else:
osd_map_f = tmpnam()
run_locally("ceph osd getmap -o {0}".format(osd_map_f))
crush_map_f = tmpnam()
if opts.subparser_name == "apply":
crush_map_txt_f = opts.crush_file
else:
assert opts.subparser_name == "interactive"
run_locally("osdmaptool --export-crush {0} {1}".format(
crush_map_f, osd_map_f))
crush_map_txt_f = tmpnam()
run_locally("crushtool -d {0} -o {1}".format(crush_map_f,
crush_map_txt_f))
run_locally("{0} {1}".format(opts.editor, crush_map_txt_f))
logger.info("Press enter, when done")
sys.stdin.readline()
if b'\x00' in open(crush_map_f, 'rb').read(1024):
run_locally("crushtool -c {0} -o {1}".format(crush_map_txt_f,
crush_map_f))
else:
print("Get already compiled crush map")
shutil.copy(crush_map_txt_f, crush_map_f)
if opts.osd_map:
# don't change original osd map file
osd_map_new_f = tmpnam()
shutil.copy(osd_map_f, osd_map_new_f)
logger.debug("Copy {0} => {1}".format(osd_map_f, osd_map_new_f))
else:
osd_map_new_f = osd_map_f
run_locally("osdmaptool --import-crush {0} {1}".format(
crush_map_f, osd_map_new_f))
osd_changes = calculate_remap(osd_map_f, osd_map_new_f, opts.pg_dump)
total_send = 0
total_moved_pg = 0
for osd_id, osd_change in sorted(osd_changes.items()):
if opts.per_osd:
print("{0:>3d}: Send: {1:>6}B".format(
osd_id, b2ssize(osd_change.bytes_out)))
print(" Recv: {0:>6}B".format(b2ssize(osd_change.bytes_in)))
print(" PG in: {0:>4d}".format(osd_change.pg_in))
print(" PG out: {0:>4d}".format(osd_change.pg_out))
total_send += osd_change.bytes_in
total_moved_pg += osd_change.pg_in
print("Total moved :", b2ssize(total_send) + "B")
print("Total PG moved :", total_moved_pg)
return 0
def request_reweight_update(node, new_weight):
osd_id = int(node.name.split('.')[1])
run_locally("ceph osd reweight {0} {1}".format(osd_id, new_weight))
def do_rebalance(config_dict, opts):
crush, curr_reweight, crushmap_bin_f, osd_map_f = load_all_data(opts)
if crush is None:
return 1
config = prepare_update_config(config_dict, crush, curr_reweight)
if config is None:
return 1
expected_weight_results = dict(
(tuple(osd.full_path), osd.weight) for osd in crush.iter_nodes('osd'))
expected_weight_results.update(
(tuple(node.full_path), new_weight)
for node, new_weight in config.rebalance_nodes)
expected_reweight_results = curr_reweight.copy()
expected_reweight_results.update(config.reweight_nodes)
if not (config.rebalance_nodes or config.reweight_nodes):
logger.info("Nothing to change")
return 0
if config.total_weight_change > 0:
logger.info("Total sum of all weight changes = %.2f",
config.total_weight_change)
if config.total_reweight_change > 0:
logger.info("Total sum of all reweight changes = %.2f",
config.total_reweight_change)
# -------------------- DO ESTIMATION ------------------------------------------------------------------------------
if not opts.no_estimate:
if config.total_reweight_change != 0.0:
logger.warning(
"Can't estimate reweight results! Estimation only includes weight changes!"
)
if config.total_weight_change == 0:
logger.info("No weight is changes. No PG/data would be moved")
else:
cmd_templ = "crushtool -i {crush_map_f} -o {crush_map_f} --update-item {id} {weight} {name} {loc}"
for node, new_weight in config.rebalance_nodes:
loc = " ".join("--loc {0} {1}".format(tp, name)
for tp, name in node.full_path)
cmd = cmd_templ.format(crush_map_f=crushmap_bin_f,
id=node.id,
weight=new_weight,
name=node.name,
loc=loc)
run_locally(cmd)
osd_map_new_f = tmpnam()
shutil.copy(osd_map_f, osd_map_new_f)
run_locally("osdmaptool --import-crush {0} {1}".format(
crushmap_bin_f, osd_map_new_f))
if opts.offline and not opts.pg_dump:
logger.warning(
"Can't calculate pg/data movement in offline mode if no pg dump provided"
)
else:
osd_changes = calculate_remap(osd_map_f,
osd_map_new_f,
pg_dump_f=opts.pg_dump)
total_send = 0
total_moved_pg = 0
for osd_id, osd_change in sorted(osd_changes.items()):
total_send += osd_change.bytes_in
total_moved_pg += osd_change.pg_in
logger.info("Total bytes to be moved : %sB",
b2ssize(total_send))
logger.info("Total PG to be moved : %s", total_moved_pg)
if opts.show_after:
osd_curr = get_osd_curr()
for osd_id, osd_change in sorted(osd_changes.items()):
pg_diff = osd_change.pg_in - osd_change.pg_out
bytes_diff = osd_change.bytes_in - osd_change.bytes_out
logger.info(
"OSD {0}, PG {1:>4d} => {2:>4d}, bytes {3:>6s} => {4:>6s}"
.format(
osd_id, osd_curr[osd_id].pg,
osd_curr[osd_id].pg + pg_diff,
b2ssize(osd_curr[osd_id].bytes),
b2ssize(osd_curr[osd_id].bytes + bytes_diff)))
if opts.estimate_only:
return 0
# -------------------- UPDATE CLUSTER -----------------------------------------------------------------------------
if opts.offline:
logger.error("No updates would be made in offline mode")
return 1
logger.info("Start updating the cluster")
already_changed = 0.0
requested_changes = config.total_weight_change / config.max_weight_change + \
config.total_reweight_change / config.max_reweight_change
wait_rebalance_to_complete(False)
round = 0
while config.rebalance_nodes or config.reweight_nodes:
if already_changed > config.min_weight_diff:
logger.info("Done %s%%",
int(already_changed * 100.0 / requested_changes + 0.5))
is_reweight_round = (round % 2 == 1 or not config.rebalance_nodes)
if is_reweight_round:
active_list = config.reweight_nodes
max_change = config.max_reweight_change
min_diff = config.min_reweight_diff
upd_func = request_reweight_update
else:
active_list = config.rebalance_nodes
max_change = config.max_weight_change
min_diff = config.min_weight_diff
upd_func = request_weight_update
next_nodes = active_list[:config.max_nodes_per_round]
del active_list[:config.max_nodes_per_round]
for node, coef in next_nodes:
change = coef - node.weight
if abs(change) > max_change + min_diff:
change = math.copysign(max_change, change)
node.weight += change
upd_func(node, node.weight)
if abs(node.weight - coef) > min_diff:
active_list.append((node, coef))
already_changed += abs(change) / max_change
wait_rebalance_to_complete(True)
round += 1
# ------------------- CHECK RESULTS --------------------------------------------------------------------------------
if not opts.verify:
return 0
logger.info("Verifying results")
crush, curr_reweight, crushmap_bin_f, osd_map_f = load_all_data(
opts, no_cache=True)
failed = False
for node in crush.iter_nodes('osd'):
path = tuple(node.full_path)
if path not in expected_weight_results:
logger.error("Can't found osd %s in expected results",
node.str_path())
failed = True
continue
if abs(expected_weight_results[path] -
node.weight) > config.min_weight_diff:
logger.error("osd %s has wrong weight %.4f expected is %.4f",
node.str_path(), node.weight,
expected_weight_results[path])
failed = True
del expected_weight_results[path]
for path in expected_weight_results:
logger.error("osd %s missed in new crush", path)
failed = True
for osd_name, curr_rw in curr_reweight.items():
if osd_name not in expected_reweight_results:
logger.error("Can't found osd %s in reweight expected results",
osd_name)
failed = True
continue
if abs(expected_reweight_results[osd_name] -
curr_rw) > config.min_reweight_diff:
logger.error("osd %s has wrong reweight %.4f expected is %.4f",
osd_name, curr_rw,
expected_reweight_results[osd_name])
failed = True
del expected_reweight_results[osd_name]
for osd_name in expected_reweight_results:
logger.error("osd %s missed in new crush", osd_name)
failed = True
if not failed:
logger.info("Succesfully verifyed")
return 0
return 1