def get_compat_weight_set_weights(self, ms):
if not CRUSHMap.have_default_choose_args(ms.crush_dump):
# enable compat weight-set first
self.log.debug('ceph osd crush weight-set create-compat')
result = CommandResult('')
self.send_command(
result, 'mon', '',
json.dumps({
'prefix': 'osd crush weight-set create-compat',
'format': 'json',
}), '')
r, outb, outs = result.wait()
if r != 0:
self.log.error('Error creating compat weight-set')
return
result = CommandResult('')
self.send_command(
result, 'mon', '',
json.dumps({
'prefix': 'osd crush dump',
'format': 'json',
}), '')
r, outb, outs = result.wait()
if r != 0:
self.log.error('Error dumping crush map')
return
try:
crushmap = json.loads(outb)
except:
raise RuntimeError('unable to parse crush map')
else:
crushmap = ms.crush_dump
raw = CRUSHMap.get_default_choose_args(crushmap)
weight_set = {}
for b in raw:
bucket = None
for t in crushmap['buckets']:
if t['id'] == b['bucket_id']:
bucket = t
break
if not bucket:
raise RuntimeError('could not find bucket %s' % b['bucket_id'])
self.log.debug('bucket items %s' % bucket['items'])
self.log.debug('weight set %s' % b['weight_set'][0])
if len(bucket['items']) != len(b['weight_set'][0]):
raise RuntimeError(
'weight-set size does not match bucket items')
for pos in range(len(bucket['items'])):
weight_set[bucket['items'][pos]
['id']] = b['weight_set'][0][pos]
self.log.debug('weight_set weights %s' % weight_set)
return weight_set
def get_compat_weight_set_weights(self, ms):
if not CRUSHMap.have_default_choose_args(ms.crush_dump):
# enable compat weight-set first
self.log.debug('ceph osd crush weight-set create-compat')
result = CommandResult('')
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'osd crush weight-set create-compat',
'format': 'json',
}), '')
r, outb, outs = result.wait()
if r != 0:
self.log.error('Error creating compat weight-set')
return
result = CommandResult('')
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'osd crush dump',
'format': 'json',
}), '')
r, outb, outs = result.wait()
if r != 0:
self.log.error('Error dumping crush map')
return
try:
crushmap = json.loads(outb)
except:
raise RuntimeError('unable to parse crush map')
else:
crushmap = ms.crush_dump
raw = CRUSHMap.get_default_choose_args(crushmap)
weight_set = {}
for b in raw:
bucket = None
for t in crushmap['buckets']:
if t['id'] == b['bucket_id']:
bucket = t
break
if not bucket:
raise RuntimeError('could not find bucket %s' % b['bucket_id'])
self.log.debug('bucket items %s' % bucket['items'])
self.log.debug('weight set %s' % b['weight_set'][0])
if len(bucket['items']) != len(b['weight_set'][0]):
raise RuntimeError('weight-set size does not match bucket items')
for pos in range(len(bucket['items'])):
weight_set[bucket['items'][pos]['id']] = b['weight_set'][0][pos]
self.log.debug('weight_set weights %s' % weight_set)
return weight_set
def show(self):
ls = []
ls.append('# starting osdmap epoch %d' %
self.initial.osdmap.get_epoch())
ls.append('# starting crush version %d' %
self.initial.osdmap.get_crush_version())
ls.append('# mode %s' % self.mode)
if len(self.compat_ws) and \
not CRUSHMap.have_default_choose_args(self.initial.crush_dump):
ls.append('ceph osd crush weight-set create-compat')
for osd, weight in self.compat_ws.iteritems():
ls.append('ceph osd crush weight-set reweight-compat %s %f' %
(osd, weight))
for osd, weight in self.osd_weights.iteritems():
ls.append('ceph osd reweight osd.%d %f' % (osd, weight))
incdump = self.inc.dump()
for pgid in incdump.get('old_pg_upmap_items', []):
ls.append('ceph osd rm-pg-upmap-items %s' % pgid)
for item in incdump.get('new_pg_upmap_items', []):
osdlist = []
for m in item['mappings']:
osdlist += [m['from'], m['to']]
ls.append('ceph osd pg-upmap-items %s %s' %
(item['pgid'], ' '.join([str(a) for a in osdlist])))
return '\n'.join(ls)
def identify_subtrees_and_overlaps(self,
osdmap: OSDMap,
crush: CRUSHMap,
result: Dict[int, CrushSubtreeResourceStatus],
overlapped_roots: Set[int],
roots: List[CrushSubtreeResourceStatus]) -> \
Tuple[List[CrushSubtreeResourceStatus],
Set[int]]:
# We identify subtrees and overlapping roots from osdmap
for pool_id, pool in osdmap.get_pools().items():
crush_rule = crush.get_rule_by_id(pool['crush_rule'])
assert crush_rule is not None
cr_name = crush_rule['rule_name']
root_id = crush.get_rule_root(cr_name)
assert root_id is not None
osds = set(crush.get_osds_under(root_id))
# Are there overlapping roots?
s = None
for prev_root_id, prev in result.items():
if osds & prev.osds:
s = prev
if prev_root_id != root_id:
overlapped_roots.add(prev_root_id)
overlapped_roots.add(root_id)
self.log.error('pool %d has overlapping roots: %s',
pool_id, overlapped_roots)
break
if not s:
s = CrushSubtreeResourceStatus()
roots.append(s)
result[root_id] = s
s.root_ids.append(root_id)
s.osds |= osds
s.pool_ids.append(pool_id)
s.pool_names.append(pool['pool_name'])
s.pg_current += pool['pg_num_target'] * pool['size']
target_ratio = pool['options'].get('target_size_ratio', 0.0)
if target_ratio:
s.total_target_ratio += target_ratio
else:
target_bytes = pool['options'].get('target_size_bytes', 0)
if target_bytes:
s.total_target_bytes += target_bytes * osdmap.pool_raw_used_rate(
pool_id)
return roots, overlapped_roots
def show(self):
ls = []
ls.append('# starting osdmap epoch %d' % self.initial.osdmap.get_epoch())
ls.append('# starting crush version %d' %
self.initial.osdmap.get_crush_version())
ls.append('# mode %s' % self.mode)
if len(self.compat_ws) and \
not CRUSHMap.have_default_choose_args(self.initial.crush_dump):
ls.append('ceph osd crush weight-set create-compat')
for osd, weight in six.iteritems(self.compat_ws):
ls.append('ceph osd crush weight-set reweight-compat %s %f' %
(osd, weight))
for osd, weight in six.iteritems(self.osd_weights):
ls.append('ceph osd reweight osd.%d %f' % (osd, weight))
incdump = self.inc.dump()
for pgid in incdump.get('old_pg_upmap_items', []):
ls.append('ceph osd rm-pg-upmap-items %s' % pgid)
for item in incdump.get('new_pg_upmap_items', []):
osdlist = []
for m in item['mappings']:
osdlist += [m['from'], m['to']]
ls.append('ceph osd pg-upmap-items %s %s' %
(item['pgid'], ' '.join([str(a) for a in osdlist])))
return '\n'.join(ls)
def execute(self, plan):
self.log.info('Executing plan %s' % plan.name)
commands = []
# compat weight-set
if len(plan.compat_ws) and \
not CRUSHMap.have_default_choose_args(plan.initial.crush_dump):
self.log.debug('ceph osd crush weight-set create-compat')
result = CommandResult('')
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'osd crush weight-set create-compat',
'format': 'json',
}), '')
r, outb, outs = result.wait()
if r != 0:
self.log.error('Error creating compat weight-set')
return r, outs
for osd, weight in six.iteritems(plan.compat_ws):
self.log.info('ceph osd crush weight-set reweight-compat osd.%d %f',
osd, weight)
result = CommandResult('')
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'osd crush weight-set reweight-compat',
'format': 'json',
'item': 'osd.%d' % osd,
'weight': [weight],
}), '')
commands.append(result)
# new_weight
reweightn = {}
for osd, weight in six.iteritems(plan.osd_weights):
reweightn[str(osd)] = str(int(weight * float(0x10000)))
if len(reweightn):
self.log.info('ceph osd reweightn %s', reweightn)
result = CommandResult('')
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'osd reweightn',
'format': 'json',
'weights': json.dumps(reweightn),
}), '')
commands.append(result)
# upmap
incdump = plan.inc.dump()
for pgid in incdump.get('old_pg_upmap_items', []):
self.log.info('ceph osd rm-pg-upmap-items %s', pgid)
result = CommandResult('foo')
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'osd rm-pg-upmap-items',
'format': 'json',
'pgid': pgid,
}), 'foo')
commands.append(result)
for item in incdump.get('new_pg_upmap_items', []):
self.log.info('ceph osd pg-upmap-items %s mappings %s', item['pgid'],
item['mappings'])
osdlist = []
for m in item['mappings']:
osdlist += [m['from'], m['to']]
result = CommandResult('foo')
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'osd pg-upmap-items',
'format': 'json',
'pgid': item['pgid'],
'id': osdlist,
}), 'foo')
commands.append(result)
# wait for commands
self.log.debug('commands %s' % commands)
for result in commands:
r, outb, outs = result.wait()
if r != 0:
self.log.error('execute error: r = %d, detail = %s' % (r, outs))
return r, outs
self.log.debug('done')
return 0, ''
def _get_pool_pg_targets(
self,
osdmap: OSDMap,
pools: Dict[str, Dict[str, Any]],
crush_map: CRUSHMap,
root_map: Dict[int, CrushSubtreeResourceStatus],
pool_stats: Dict[int, Dict[str, int]],
ret: List[Dict[str, Any]],
threshold: float,
func_pass: '******',
overlapped_roots: Set[int],
) -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]], Dict[str, Dict[
str, Any]]]:
"""
Calculates final_pg_target of each pools and determine if it needs
scaling, this depends on the profile of the autoscaler. For scale-down,
we start out with a full complement of pgs and only descrease it when other
pools needs more pgs due to increased usage. For scale-up, we start out with
the minimal amount of pgs and only scale when there is increase in usage.
"""
even_pools: Dict[str, Dict[str, Any]] = {}
bulk_pools: Dict[str, Dict[str, Any]] = {}
for pool_name, p in pools.items():
pool_id = p['pool']
if pool_id not in pool_stats:
# race with pool deletion; skip
continue
# FIXME: we assume there is only one take per pool, but that
# may not be true.
crush_rule = crush_map.get_rule_by_id(p['crush_rule'])
assert crush_rule is not None
cr_name = crush_rule['rule_name']
root_id = crush_map.get_rule_root(cr_name)
assert root_id is not None
if root_id in overlapped_roots:
# skip pools
# with overlapping roots
self.log.warn(
"pool %d contains an overlapping root %d"
"... skipping scaling", pool_id, root_id)
continue
capacity = root_map[root_id].capacity
assert capacity is not None
if capacity == 0:
self.log.debug('skipping empty subtree %s', cr_name)
continue
raw_used_rate = osdmap.pool_raw_used_rate(pool_id)
pool_logical_used = pool_stats[pool_id]['stored']
bias = p['options'].get('pg_autoscale_bias', 1.0)
target_bytes = 0
# ratio takes precedence if both are set
if p['options'].get('target_size_ratio', 0.0) == 0.0:
target_bytes = p['options'].get('target_size_bytes', 0)
# What proportion of space are we using?
actual_raw_used = pool_logical_used * raw_used_rate
actual_capacity_ratio = float(actual_raw_used) / capacity
pool_raw_used = max(pool_logical_used,
target_bytes) * raw_used_rate
capacity_ratio = float(pool_raw_used) / capacity
self.log.info("effective_target_ratio {0} {1} {2} {3}".format(
p['options'].get('target_size_ratio',
0.0), root_map[root_id].total_target_ratio,
root_map[root_id].total_target_bytes, capacity))
target_ratio = effective_target_ratio(
p['options'].get('target_size_ratio',
0.0), root_map[root_id].total_target_ratio,
root_map[root_id].total_target_bytes, capacity)
# determine if the pool is a bulk
bulk = False
flags = p['flags_names'].split(",")
if "bulk" in flags:
bulk = True
capacity_ratio = max(capacity_ratio, target_ratio)
final_ratio, pool_pg_target, final_pg_target = self._calc_final_pg_target(
p, pool_name, root_map, root_id, capacity_ratio, bias,
even_pools, bulk_pools, func_pass, bulk)
if final_ratio is None:
continue
adjust = False
if (final_pg_target > p['pg_num_target'] * threshold or
final_pg_target < p['pg_num_target'] / threshold) and \
final_ratio >= 0.0 and \
final_ratio <= 1.0:
adjust = True
assert pool_pg_target is not None
ret.append({
'pool_id':
pool_id,
'pool_name':
p['pool_name'],
'crush_root_id':
root_id,
'pg_autoscale_mode':
p['pg_autoscale_mode'],
'pg_num_target':
p['pg_num_target'],
'logical_used':
pool_logical_used,
'target_bytes':
target_bytes,
'raw_used_rate':
raw_used_rate,
'subtree_capacity':
capacity,
'actual_raw_used':
actual_raw_used,
'raw_used':
pool_raw_used,
'actual_capacity_ratio':
actual_capacity_ratio,
'capacity_ratio':
capacity_ratio,
'target_ratio':
p['options'].get('target_size_ratio', 0.0),
'effective_target_ratio':
target_ratio,
'pg_num_ideal':
int(pool_pg_target),
'pg_num_final':
final_pg_target,
'would_adjust':
adjust,
'bias':
p.get('options', {}).get('pg_autoscale_bias', 1.0),
'bulk':
bulk,
})
return ret, bulk_pools, even_pools
def get_subtree_resource_status(self,
osdmap: OSDMap,
crush: CRUSHMap) -> Tuple[Dict[int, CrushSubtreeResourceStatus],
Dict[int, int]]:
"""
For each CRUSH subtree of interest (i.e. the roots under which
we have pools), calculate the current resource usages and targets,
such as how many PGs there are, vs. how many PGs we would
like there to be.
"""
result: Dict[int, CrushSubtreeResourceStatus] = {}
pool_root = {}
roots = []
# identify subtrees (note that they may overlap!)
for pool_id, pool in osdmap.get_pools().items():
crush_rule = crush.get_rule_by_id(pool['crush_rule'])
assert crush_rule is not None
cr_name = crush_rule['rule_name']
root_id = crush.get_rule_root(cr_name)
assert root_id is not None
pool_root[pool_id] = root_id
osds = set(crush.get_osds_under(root_id))
# do we intersect an existing root?
s = None
for prev in result.values():
if osds & prev.osds:
s = prev
break
if not s:
s = CrushSubtreeResourceStatus()
roots.append(s)
result[root_id] = s
s.root_ids.append(root_id)
s.osds |= osds
s.pool_ids.append(pool_id)
s.pool_names.append(pool['pool_name'])
s.pg_current += pool['pg_num_target'] * pool['size']
target_ratio = pool['options'].get('target_size_ratio', 0.0)
if target_ratio:
s.total_target_ratio += target_ratio
else:
target_bytes = pool['options'].get('target_size_bytes', 0)
if target_bytes:
s.total_target_bytes += target_bytes * osdmap.pool_raw_used_rate(pool_id)
# finish subtrees
all_stats = self.get('osd_stats')
for s in roots:
assert s.osds is not None
s.osd_count = len(s.osds)
s.pg_target = s.osd_count * self.mon_target_pg_per_osd
s.pg_left = s.pg_target
s.pool_count = len(s.pool_ids)
capacity = 0
for osd_stats in all_stats['osd_stats']:
if osd_stats['osd'] in s.osds:
# Intentionally do not apply the OSD's reweight to
# this, because we want to calculate PG counts based
# on the physical storage available, not how it is
# reweighted right now.
capacity += osd_stats['kb'] * 1024
s.capacity = capacity
self.log.debug('root_ids %s pools %s with %d osds, pg_target %d',
s.root_ids,
s.pool_ids,
s.osd_count,
s.pg_target)
return result, pool_root
