def get_options(self):
return [
Option(
name=self.OPTION_FMT.format(feature),
default=(feature not in PREDISABLED_FEATURES),
type='bool',
) for feature in Features
]
class PgAutoscaler(MgrModule):
"""
PG autoscaler.
"""
NATIVE_OPTIONS = [
'mon_target_pg_per_osd',
'mon_max_pg_per_osd',
]
MODULE_OPTIONS = [
Option(name='sleep_interval', type='secs', default=60),
Option(
name='threshold',
type='float',
desc='scaling threshold',
long_desc=
('The factor by which the `NEW PG_NUM` must vary from the current'
'`PG_NUM` before being accepted. Cannot be less than 1.0'),
default=3.0,
min=1.0),
]
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(PgAutoscaler, self).__init__(*args, **kwargs)
self._shutdown = threading.Event()
self._event: Dict[int, PgAdjustmentProgress] = {}
# So much of what we do peeks at the osdmap that it's easiest
# to just keep a copy of the pythonized version.
self._osd_map = None
if TYPE_CHECKING:
self.sleep_interval = 60
self.mon_target_pg_per_osd = 0
self.threshold = 3.0
def config_notify(self) -> None:
for opt in self.NATIVE_OPTIONS:
setattr(self, opt, self.get_ceph_option(opt))
self.log.debug(' native option %s = %s', opt, getattr(self, opt))
for opt in self.MODULE_OPTIONS:
setattr(self, opt['name'], self.get_module_option(opt['name']))
self.log.debug(' mgr option %s = %s', opt['name'],
getattr(self, opt['name']))
@CLIReadCommand('osd pool autoscale-status')
def _command_autoscale_status(self,
format: str = 'plain'
) -> Tuple[int, str, str]:
"""
report on pool pg_num sizing recommendation and intent
"""
osdmap = self.get_osdmap()
pools = osdmap.get_pools_by_name()
ps, root_map = self._get_pool_status(osdmap, pools)
if format in ('json', 'json-pretty'):
return 0, json.dumps(ps, indent=4, sort_keys=True), ''
else:
table = PrettyTable(
[
'POOL',
'SIZE',
'TARGET SIZE',
'RATE',
'RAW CAPACITY',
'RATIO',
'TARGET RATIO',
'EFFECTIVE RATIO',
'BIAS',
'PG_NUM',
# 'IDEAL',
'NEW PG_NUM',
'AUTOSCALE',
'BULK'
],
border=False)
table.left_padding_width = 0
table.right_padding_width = 2
table.align['POOL'] = 'l'
table.align['SIZE'] = 'r'
table.align['TARGET SIZE'] = 'r'
table.align['RATE'] = 'r'
table.align['RAW CAPACITY'] = 'r'
table.align['RATIO'] = 'r'
table.align['TARGET RATIO'] = 'r'
table.align['EFFECTIVE RATIO'] = 'r'
table.align['BIAS'] = 'r'
table.align['PG_NUM'] = 'r'
# table.align['IDEAL'] = 'r'
table.align['NEW PG_NUM'] = 'r'
table.align['AUTOSCALE'] = 'l'
table.align['BULK'] = 'l'
for p in ps:
if p['would_adjust']:
final = str(p['pg_num_final'])
else:
final = ''
if p['target_bytes'] > 0:
ts = mgr_util.format_bytes(p['target_bytes'], 6)
else:
ts = ''
if p['target_ratio'] > 0.0:
tr = '%.4f' % p['target_ratio']
else:
tr = ''
if p['effective_target_ratio'] > 0.0:
etr = '%.4f' % p['effective_target_ratio']
else:
etr = ''
table.add_row([
p['pool_name'],
mgr_util.format_bytes(p['logical_used'], 6),
ts,
p['raw_used_rate'],
mgr_util.format_bytes(p['subtree_capacity'], 6),
'%.4f' % p['capacity_ratio'],
tr,
etr,
p['bias'],
p['pg_num_target'],
# p['pg_num_ideal'],
final,
p['pg_autoscale_mode'],
str(p['bulk'])
])
return 0, table.get_string(), ''
@CLIWriteCommand("osd pool set threshold")
def set_scaling_threshold(self, num: float) -> Tuple[int, str, str]:
"""
set the autoscaler threshold
A.K.A. the factor by which the new PG_NUM must vary from the existing PG_NUM
"""
if num < 1.0:
return 22, "", "threshold cannot be set less than 1.0"
self.set_module_option("threshold", num)
return 0, "threshold updated", ""
def serve(self) -> None:
self.config_notify()
while not self._shutdown.is_set():
self._maybe_adjust()
self._update_progress_events()
self._shutdown.wait(timeout=self.sleep_interval)
def shutdown(self) -> None:
self.log.info('Stopping pg_autoscaler')
self._shutdown.set()
def identify_subtrees_and_overlaps(self,
osdmap: OSDMap,
crush: CRUSHMap,
result: Dict[int, CrushSubtreeResourceStatus],
overlapped_roots: Set[int],
roots: List[CrushSubtreeResourceStatus]) -> \
Tuple[List[CrushSubtreeResourceStatus],
Set[int]]:
# We identify subtrees and overlapping roots from osdmap
for pool_id, pool in osdmap.get_pools().items():
crush_rule = crush.get_rule_by_id(pool['crush_rule'])
assert crush_rule is not None
cr_name = crush_rule['rule_name']
root_id = crush.get_rule_root(cr_name)
assert root_id is not None
osds = set(crush.get_osds_under(root_id))
# Are there overlapping roots?
s = None
for prev_root_id, prev in result.items():
if osds & prev.osds:
s = prev
if prev_root_id != root_id:
overlapped_roots.add(prev_root_id)
overlapped_roots.add(root_id)
self.log.error('pool %d has overlapping roots: %s',
pool_id, overlapped_roots)
break
if not s:
s = CrushSubtreeResourceStatus()
roots.append(s)
result[root_id] = s
s.root_ids.append(root_id)
s.osds |= osds
s.pool_ids.append(pool_id)
s.pool_names.append(pool['pool_name'])
s.pg_current += pool['pg_num_target'] * pool['size']
target_ratio = pool['options'].get('target_size_ratio', 0.0)
if target_ratio:
s.total_target_ratio += target_ratio
else:
target_bytes = pool['options'].get('target_size_bytes', 0)
if target_bytes:
s.total_target_bytes += target_bytes * osdmap.pool_raw_used_rate(
pool_id)
return roots, overlapped_roots
def get_subtree_resource_status(
self, osdmap: OSDMap, crush: CRUSHMap
) -> Tuple[Dict[int, CrushSubtreeResourceStatus], Set[int]]:
"""
For each CRUSH subtree of interest (i.e. the roots under which
we have pools), calculate the current resource usages and targets,
such as how many PGs there are, vs. how many PGs we would
like there to be.
"""
result: Dict[int, CrushSubtreeResourceStatus] = {}
roots: List[CrushSubtreeResourceStatus] = []
overlapped_roots: Set[int] = set()
# identify subtrees and overlapping roots
roots, overlapped_roots = self.identify_subtrees_and_overlaps(
osdmap, crush, result, overlapped_roots, roots)
# finish subtrees
all_stats = self.get('osd_stats')
for s in roots:
assert s.osds is not None
s.osd_count = len(s.osds)
s.pg_target = s.osd_count * self.mon_target_pg_per_osd
s.pg_left = s.pg_target
s.pool_count = len(s.pool_ids)
capacity = 0
for osd_stats in all_stats['osd_stats']:
if osd_stats['osd'] in s.osds:
# Intentionally do not apply the OSD's reweight to
# this, because we want to calculate PG counts based
# on the physical storage available, not how it is
# reweighted right now.
capacity += osd_stats['kb'] * 1024
s.capacity = capacity
self.log.debug('root_ids %s pools %s with %d osds, pg_target %d',
s.root_ids, s.pool_ids, s.osd_count, s.pg_target)
return result, overlapped_roots
def _calc_final_pg_target(
self,
p: Dict[str, Any],
pool_name: str,
root_map: Dict[int, CrushSubtreeResourceStatus],
root_id: int,
capacity_ratio: float,
bias: float,
even_pools: Dict[str, Dict[str, Any]],
bulk_pools: Dict[str, Dict[str, Any]],
func_pass: '******',
bulk: bool,
) -> Union[Tuple[float, int, int], Tuple[None, None, None]]:
"""
`profile` determines behaviour of the autoscaler.
`first_pass` flag used to determine if this is the first
pass where the caller tries to calculate/adjust pools that has
used_ratio > even_ratio else this is the second pass,
we calculate final_ratio by giving it 1 / pool_count
of the root we are currently looking at.
"""
if func_pass == 'first':
# first pass to deal with small pools (no bulk flag)
# calculating final_pg_target based on capacity ratio
# we also keep track of bulk_pools to be used in second pass
if not bulk:
final_ratio = capacity_ratio
pg_left = root_map[root_id].pg_left
assert pg_left is not None
used_pg = final_ratio * pg_left
root_map[root_id].pg_left -= int(used_pg)
root_map[root_id].pool_used += 1
pool_pg_target = used_pg / p['size'] * bias
else:
bulk_pools[pool_name] = p
return None, None, None
elif func_pass == 'second':
# second pass we calculate the final_pg_target
# for pools that have used_ratio > even_ratio
# and we keep track of even pools to be used in third pass
pool_count = root_map[root_id].pool_count
assert pool_count is not None
even_ratio = 1 / (pool_count - root_map[root_id].pool_used)
used_ratio = capacity_ratio
if used_ratio > even_ratio:
root_map[root_id].pool_used += 1
else:
even_pools[pool_name] = p
return None, None, None
final_ratio = max(used_ratio, even_ratio)
pg_left = root_map[root_id].pg_left
assert pg_left is not None
used_pg = final_ratio * pg_left
root_map[root_id].pg_left -= int(used_pg)
pool_pg_target = used_pg / p['size'] * bias
else:
# third pass we just split the pg_left to all even_pools
pool_count = root_map[root_id].pool_count
assert pool_count is not None
final_ratio = 1 / (pool_count - root_map[root_id].pool_used)
pool_pg_target = (final_ratio *
root_map[root_id].pg_left) / p['size'] * bias
final_pg_target = max(
p.get('options', {}).get('pg_num_min', PG_NUM_MIN),
nearest_power_of_two(pool_pg_target))
self.log.info("Pool '{0}' root_id {1} using {2} of space, bias {3}, "
"pg target {4} quantized to {5} (current {6})".format(
p['pool_name'], root_id, capacity_ratio, bias,
pool_pg_target, final_pg_target, p['pg_num_target']))
return final_ratio, pool_pg_target, final_pg_target
def _get_pool_pg_targets(
self,
osdmap: OSDMap,
pools: Dict[str, Dict[str, Any]],
crush_map: CRUSHMap,
root_map: Dict[int, CrushSubtreeResourceStatus],
pool_stats: Dict[int, Dict[str, int]],
ret: List[Dict[str, Any]],
threshold: float,
func_pass: '******',
overlapped_roots: Set[int],
) -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]], Dict[str, Dict[
str, Any]]]:
"""
Calculates final_pg_target of each pools and determine if it needs
scaling, this depends on the profile of the autoscaler. For scale-down,
we start out with a full complement of pgs and only descrease it when other
pools needs more pgs due to increased usage. For scale-up, we start out with
the minimal amount of pgs and only scale when there is increase in usage.
"""
even_pools: Dict[str, Dict[str, Any]] = {}
bulk_pools: Dict[str, Dict[str, Any]] = {}
for pool_name, p in pools.items():
pool_id = p['pool']
if pool_id not in pool_stats:
# race with pool deletion; skip
continue
# FIXME: we assume there is only one take per pool, but that
# may not be true.
crush_rule = crush_map.get_rule_by_id(p['crush_rule'])
assert crush_rule is not None
cr_name = crush_rule['rule_name']
root_id = crush_map.get_rule_root(cr_name)
assert root_id is not None
if root_id in overlapped_roots:
# skip pools
# with overlapping roots
self.log.warn(
"pool %d contains an overlapping root %d"
"... skipping scaling", pool_id, root_id)
continue
capacity = root_map[root_id].capacity
assert capacity is not None
if capacity == 0:
self.log.debug('skipping empty subtree %s', cr_name)
continue
raw_used_rate = osdmap.pool_raw_used_rate(pool_id)
pool_logical_used = pool_stats[pool_id]['stored']
bias = p['options'].get('pg_autoscale_bias', 1.0)
target_bytes = 0
# ratio takes precedence if both are set
if p['options'].get('target_size_ratio', 0.0) == 0.0:
target_bytes = p['options'].get('target_size_bytes', 0)
# What proportion of space are we using?
actual_raw_used = pool_logical_used * raw_used_rate
actual_capacity_ratio = float(actual_raw_used) / capacity
pool_raw_used = max(pool_logical_used,
target_bytes) * raw_used_rate
capacity_ratio = float(pool_raw_used) / capacity
self.log.info("effective_target_ratio {0} {1} {2} {3}".format(
p['options'].get('target_size_ratio',
0.0), root_map[root_id].total_target_ratio,
root_map[root_id].total_target_bytes, capacity))
target_ratio = effective_target_ratio(
p['options'].get('target_size_ratio',
0.0), root_map[root_id].total_target_ratio,
root_map[root_id].total_target_bytes, capacity)
# determine if the pool is a bulk
bulk = False
flags = p['flags_names'].split(",")
if "bulk" in flags:
bulk = True
capacity_ratio = max(capacity_ratio, target_ratio)
final_ratio, pool_pg_target, final_pg_target = self._calc_final_pg_target(
p, pool_name, root_map, root_id, capacity_ratio, bias,
even_pools, bulk_pools, func_pass, bulk)
if final_ratio is None:
continue
adjust = False
if (final_pg_target > p['pg_num_target'] * threshold or
final_pg_target < p['pg_num_target'] / threshold) and \
final_ratio >= 0.0 and \
final_ratio <= 1.0:
adjust = True
assert pool_pg_target is not None
ret.append({
'pool_id':
pool_id,
'pool_name':
p['pool_name'],
'crush_root_id':
root_id,
'pg_autoscale_mode':
p['pg_autoscale_mode'],
'pg_num_target':
p['pg_num_target'],
'logical_used':
pool_logical_used,
'target_bytes':
target_bytes,
'raw_used_rate':
raw_used_rate,
'subtree_capacity':
capacity,
'actual_raw_used':
actual_raw_used,
'raw_used':
pool_raw_used,
'actual_capacity_ratio':
actual_capacity_ratio,
'capacity_ratio':
capacity_ratio,
'target_ratio':
p['options'].get('target_size_ratio', 0.0),
'effective_target_ratio':
target_ratio,
'pg_num_ideal':
int(pool_pg_target),
'pg_num_final':
final_pg_target,
'would_adjust':
adjust,
'bias':
p.get('options', {}).get('pg_autoscale_bias', 1.0),
'bulk':
bulk,
})
return ret, bulk_pools, even_pools
def _get_pool_status(
self,
osdmap: OSDMap,
pools: Dict[str, Dict[str, Any]],
) -> Tuple[List[Dict[str, Any]], Dict[int, CrushSubtreeResourceStatus]]:
threshold = self.threshold
assert threshold >= 1.0
crush_map = osdmap.get_crush()
root_map, overlapped_roots = self.get_subtree_resource_status(
osdmap, crush_map)
df = self.get('df')
pool_stats = dict([(p['id'], p['stats']) for p in df['pools']])
ret: List[Dict[str, Any]] = []
# Iterate over all pools to determine how they should be sized.
# First call of _get_pool_pg_targets() is to find/adjust pools that uses more capacaity than
# the even_ratio of other pools and we adjust those first.
# Second call make use of the even_pools we keep track of in the first call.
# All we need to do is iterate over those and give them 1/pool_count of the
# total pgs.
ret, bulk_pools, _ = self._get_pool_pg_targets(osdmap, pools,
crush_map, root_map,
pool_stats, ret,
threshold, 'first',
overlapped_roots)
ret, _, even_pools = self._get_pool_pg_targets(osdmap, bulk_pools,
crush_map, root_map,
pool_stats, ret,
threshold, 'second',
overlapped_roots)
ret, _, _ = self._get_pool_pg_targets(osdmap, even_pools, crush_map,
root_map, pool_stats, ret,
threshold, 'third',
overlapped_roots)
return (ret, root_map)
def _update_progress_events(self) -> None:
osdmap = self.get_osdmap()
pools = osdmap.get_pools()
for pool_id in list(self._event):
ev = self._event[pool_id]
pool_data = pools.get(pool_id)
if pool_data is None or pool_data['pg_num'] == pool_data[
'pg_num_target'] or ev.pg_num == ev.pg_num_target:
# pool is gone or we've reached our target
self.remote('progress', 'complete', ev.ev_id)
del self._event[pool_id]
continue
ev.update(self, (ev.pg_num - pool_data['pg_num']) /
(ev.pg_num - ev.pg_num_target))
def _maybe_adjust(self) -> None:
self.log.info('_maybe_adjust')
osdmap = self.get_osdmap()
if osdmap.get_require_osd_release() < 'nautilus':
return
pools = osdmap.get_pools_by_name()
ps, root_map = self._get_pool_status(osdmap, pools)
# Anyone in 'warn', set the health message for them and then
# drop them from consideration.
too_few = []
too_many = []
bytes_and_ratio = []
health_checks: Dict[str, Dict[str, Union[int, str, List[str]]]] = {}
total_bytes = dict([(r, 0) for r in iter(root_map)])
total_target_bytes = dict([(r, 0.0) for r in iter(root_map)])
target_bytes_pools: Dict[int,
List[int]] = dict([(r, [])
for r in iter(root_map)])
for p in ps:
pool_id = p['pool_id']
pool_opts = pools[p['pool_name']]['options']
if pool_opts.get('target_size_ratio', 0) > 0 and pool_opts.get(
'target_size_bytes', 0) > 0:
bytes_and_ratio.append(
'Pool %s has target_size_bytes and target_size_ratio set' %
p['pool_name'])
total_bytes[p['crush_root_id']] += max(
p['actual_raw_used'], p['target_bytes'] * p['raw_used_rate'])
if p['target_bytes'] > 0:
total_target_bytes[p[
'crush_root_id']] += p['target_bytes'] * p['raw_used_rate']
target_bytes_pools[p['crush_root_id']].append(p['pool_name'])
if not p['would_adjust']:
continue
if p['pg_autoscale_mode'] == 'warn':
msg = 'Pool %s has %d placement groups, should have %d' % (
p['pool_name'], p['pg_num_target'], p['pg_num_final'])
if p['pg_num_final'] > p['pg_num_target']:
too_few.append(msg)
else:
too_many.append(msg)
if p['pg_autoscale_mode'] == 'on':
# Note that setting pg_num actually sets pg_num_target (see
# OSDMonitor.cc)
r = self.mon_command({
'prefix': 'osd pool set',
'pool': p['pool_name'],
'var': 'pg_num',
'val': str(p['pg_num_final'])
})
# create new event or update existing one to reflect
# progress from current state to the new pg_num_target
pool_data = pools[p['pool_name']]
pg_num = pool_data['pg_num']
new_target = p['pg_num_final']
if pool_id in self._event:
self._event[pool_id].reset(pg_num, new_target)
else:
self._event[pool_id] = PgAdjustmentProgress(
pool_id, pg_num, new_target)
self._event[pool_id].update(self, 0.0)
if r[0] != 0:
# FIXME: this is a serious and unexpected thing,
# we should expose it as a cluster log error once
# the hook for doing that from ceph-mgr modules is
# in.
self.log.error(
"pg_num adjustment on {0} to {1} failed: {2}".format(
p['pool_name'], p['pg_num_final'], r))
if too_few:
summary = "{0} pools have too few placement groups".format(
len(too_few))
health_checks['POOL_TOO_FEW_PGS'] = {
'severity': 'warning',
'summary': summary,
'count': len(too_few),
'detail': too_few
}
if too_many:
summary = "{0} pools have too many placement groups".format(
len(too_many))
health_checks['POOL_TOO_MANY_PGS'] = {
'severity': 'warning',
'summary': summary,
'count': len(too_many),
'detail': too_many
}
too_much_target_bytes = []
for root_id, total in total_bytes.items():
total_target = int(total_target_bytes[root_id])
capacity = root_map[root_id].capacity
assert capacity is not None
if total_target > 0 and total > capacity and capacity:
too_much_target_bytes.append(
'Pools %s overcommit available storage by %.03fx due to '
'target_size_bytes %s on pools %s' %
(root_map[root_id].pool_names, total / capacity,
mgr_util.format_bytes(total_target, 5, colored=False),
target_bytes_pools[root_id]))
elif total_target > capacity and capacity:
too_much_target_bytes.append(
'Pools %s overcommit available storage by %.03fx due to '
'collective target_size_bytes of %s' % (
root_map[root_id].pool_names,
total / capacity,
mgr_util.format_bytes(total_target, 5, colored=False),
))
if too_much_target_bytes:
health_checks['POOL_TARGET_SIZE_BYTES_OVERCOMMITTED'] = {
'severity':
'warning',
'summary':
"%d subtrees have overcommitted pool target_size_bytes" %
len(too_much_target_bytes),
'count':
len(too_much_target_bytes),
'detail':
too_much_target_bytes,
}
if bytes_and_ratio:
health_checks['POOL_HAS_TARGET_SIZE_BYTES_AND_RATIO'] = {
'severity':
'warning',
'summary':
"%d pools have both target_size_bytes and target_size_ratio set"
% len(bytes_and_ratio),
'count':
len(bytes_and_ratio),
'detail':
bytes_and_ratio,
}
self.set_health_checks(health_checks)
class Module(MgrModule, CherryPyConfig):
"""
dashboard module entrypoint
"""
COMMANDS = [
{
'cmd': 'dashboard set-jwt-token-ttl '
'name=seconds,type=CephInt',
'desc': 'Set the JWT token TTL in seconds',
'perm': 'w'
},
{
'cmd': 'dashboard get-jwt-token-ttl',
'desc': 'Get the JWT token TTL in seconds',
'perm': 'r'
},
{
"cmd": "dashboard create-self-signed-cert",
"desc": "Create self signed certificate",
"perm": "w"
},
]
COMMANDS.extend(options_command_list())
COMMANDS.extend(SSO_COMMANDS)
PLUGIN_MANAGER.hook.register_commands()
MODULE_OPTIONS = [
Option(name='server_addr', type='str', default='::'),
Option(name='server_port', type='int', default=8443),
Option(name='jwt_token_ttl', type='int', default=28800),
Option(name='password', type='str', default=''),
Option(name='url_prefix', type='str', default=''),
Option(name='username', type='str', default=''),
Option(name='key_file', type='str', default=''),
Option(name='crt_file', type='str', default=''),
Option(name='ssl', type='bool', default=True)
]
MODULE_OPTIONS.extend(options_schema_list())
for options in PLUGIN_MANAGER.hook.get_options() or []:
MODULE_OPTIONS.extend(options)
__pool_stats = collections.defaultdict(
lambda: collections.defaultdict(lambda: collections.deque(maxlen=10)))
def __init__(self, *args, **kwargs):
super(Module, self).__init__(*args, **kwargs)
CherryPyConfig.__init__(self)
mgr.init(self)
self._stopping = threading.Event()
self.shutdown_event = threading.Event()
self.ACCESS_CTRL_DB = None
self.SSO_DB = None
@classmethod
def can_run(cls):
if cherrypy is None:
return False, "Missing dependency: cherrypy"
if not os.path.exists(cls.get_frontend_path()):
return False, "Frontend assets not found: incomplete build?"
return True, ""
@classmethod
def get_frontend_path(cls):
current_dir = os.path.dirname(os.path.abspath(__file__))
return os.path.join(current_dir, 'frontend/dist')
def serve(self):
AuthManager.initialize()
load_sso_db()
uri = self.await_configuration()
if uri is None:
# We were shut down while waiting
return
# Publish the URI that others may use to access the service we're
# about to start serving
self.set_uri(uri)
mapper, parent_urls = generate_routes(self.url_prefix)
config = {
self.url_prefix or '/': {
'tools.staticdir.on': True,
'tools.staticdir.dir': self.get_frontend_path(),
'tools.staticdir.index': 'index.html'
}
}
for purl in parent_urls:
config[purl] = {'request.dispatch': mapper}
cherrypy.tree.mount(None, config=config)
PLUGIN_MANAGER.hook.setup()
cherrypy.engine.start()
NotificationQueue.start_queue()
TaskManager.init()
logger.info('Engine started.')
# wait for the shutdown event
self.shutdown_event.wait()
self.shutdown_event.clear()
NotificationQueue.stop()
cherrypy.engine.stop()
logger.info('Engine stopped')
def shutdown(self):
super(Module, self).shutdown()
CherryPyConfig.shutdown(self)
logger.info('Stopping engine...')
self.shutdown_event.set()
def handle_command(self, inbuf, cmd):
# pylint: disable=too-many-return-statements
res = handle_option_command(cmd)
if res[0] != -errno.ENOSYS:
return res
res = handle_sso_command(cmd)
if res[0] != -errno.ENOSYS:
return res
if cmd['prefix'] == 'dashboard set-jwt-token-ttl':
self.set_module_option('jwt_token_ttl', str(cmd['seconds']))
return 0, 'JWT token TTL updated', ''
if cmd['prefix'] == 'dashboard get-jwt-token-ttl':
ttl = self.get_module_option('jwt_token_ttl',
JwtManager.JWT_TOKEN_TTL)
return 0, str(ttl), ''
if cmd['prefix'] == 'dashboard create-self-signed-cert':
self.create_self_signed_cert()
return 0, 'Self-signed certificate created', ''
return (-errno.EINVAL, '',
'Command not found \'{0}\''.format(cmd['prefix']))
def create_self_signed_cert(self):
# create a key pair
pkey = crypto.PKey()
pkey.generate_key(crypto.TYPE_RSA, 2048)
# create a self-signed cert
cert = crypto.X509()
cert.get_subject().O = "IT"
cert.get_subject().CN = "ceph-dashboard"
cert.set_serial_number(int(uuid4()))
cert.gmtime_adj_notBefore(0)
cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)
cert.set_issuer(cert.get_subject())
cert.set_pubkey(pkey)
cert.sign(pkey, 'sha512')
cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
self.set_store('crt', cert.decode('utf-8'))
pkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)
self.set_store('key', pkey.decode('utf-8'))
def notify(self, notify_type, notify_id):
NotificationQueue.new_notification(notify_type, notify_id)
def get_updated_pool_stats(self):
df = self.get('df')
pool_stats = {p['id']: p['stats'] for p in df['pools']}
now = time.time()
for pool_id, stats in pool_stats.items():
for stat_name, stat_val in stats.items():
self.__pool_stats[pool_id][stat_name].append((now, stat_val))
return self.__pool_stats
class Module(MgrModule):
run = False
config: Dict[str, OptionValue] = {}
ceph_health_mapping = {'HEALTH_OK': 0, 'HEALTH_WARN': 1, 'HEALTH_ERR': 2}
_zabbix_hosts: List[Dict[str, Union[str, int]]] = list()
@property
def config_keys(self) -> Dict[str, OptionValue]:
return dict(
(o['name'], o.get('default', None)) for o in self.MODULE_OPTIONS)
MODULE_OPTIONS = [
Option(name='zabbix_sender', default='/usr/bin/zabbix_sender'),
Option(name='zabbix_host', type='str', default=None),
Option(name='zabbix_port', type='int', default=10051),
Option(name='identifier', default=""),
Option(name='interval', type='secs', default=60),
Option(name='discovery_interval', type='uint', default=100)
]
COMMANDS = [
{
"cmd": "zabbix config-set name=key,type=CephString "
"name=value,type=CephString",
"desc": "Set a configuration value",
"perm": "rw"
},
{
"cmd": "zabbix config-show",
"desc": "Show current configuration",
"perm": "r"
},
{
"cmd": "zabbix send",
"desc": "Force sending data to Zabbix",
"perm": "rw"
},
{
"cmd": "zabbix discovery",
"desc": "Discovering Zabbix data",
"perm": "r"
},
]
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
self.event = Event()
def init_module_config(self) -> None:
self.fsid = self.get('mon_map')['fsid']
self.log.debug('Found Ceph fsid %s', self.fsid)
for key, default in self.config_keys.items():
self.set_config_option(key, self.get_module_option(key, default))
if self.config['zabbix_host']:
self._parse_zabbix_hosts()
def set_config_option(self, option: str, value: OptionValue) -> bool:
if option not in self.config_keys.keys():
raise RuntimeError('{0} is a unknown configuration '
'option'.format(option))
if option in ['zabbix_port', 'interval', 'discovery_interval']:
try:
int_value = int(value) # type: ignore
except (ValueError, TypeError):
raise RuntimeError('invalid {0} configured. Please specify '
'a valid integer'.format(option))
if option == 'interval' and int_value < 10:
raise RuntimeError('interval should be set to at least 10 seconds')
if option == 'discovery_interval' and int_value < 10:
raise RuntimeError(
"discovery_interval should not be more frequent "
"than once in 10 regular data collection")
self.log.debug('Setting in-memory config option %s to: %s', option,
value)
self.config[option] = value
return True
def _parse_zabbix_hosts(self) -> None:
self._zabbix_hosts = list()
servers = cast(str, self.config['zabbix_host']).split(",")
for server in servers:
uri = re.match(
"(?:(?:\[?)([a-z0-9-\.]+|[a-f0-9:\.]+)(?:\]?))(?:((?::))([0-9]{1,5}))?$",
server)
if uri:
zabbix_host, sep, opt_zabbix_port = uri.groups()
if sep == ':':
zabbix_port = int(opt_zabbix_port)
else:
zabbix_port = cast(int, self.config['zabbix_port'])
self._zabbix_hosts.append({
'zabbix_host': zabbix_host,
'zabbix_port': zabbix_port
})
else:
self.log.error('Zabbix host "%s" is not valid', server)
self.log.error('Parsed Zabbix hosts: %s', self._zabbix_hosts)
def get_pg_stats(self) -> Dict[str, int]:
stats = dict()
pg_states = [
'active', 'peering', 'clean', 'scrubbing', 'undersized',
'backfilling', 'recovering', 'degraded', 'inconsistent',
'remapped', 'backfill_toofull', 'backfill_wait', 'recovery_wait'
]
for state in pg_states:
stats['num_pg_{0}'.format(state)] = 0
pg_status = self.get('pg_status')
stats['num_pg'] = pg_status['num_pgs']
for state in pg_status['pgs_by_state']:
states = state['state_name'].split('+')
for s in pg_states:
key = 'num_pg_{0}'.format(s)
if s in states:
stats[key] += state['count']
return stats
def get_data(self) -> Dict[str, Union[int, float]]:
data = dict()
health = json.loads(self.get('health')['json'])
# 'status' is luminous+, 'overall_status' is legacy mode.
data['overall_status'] = health.get('status',
health.get('overall_status'))
data['overall_status_int'] = \
self.ceph_health_mapping.get(data['overall_status'])
mon_status = json.loads(self.get('mon_status')['json'])
data['num_mon'] = len(mon_status['monmap']['mons'])
df = self.get('df')
data['num_pools'] = len(df['pools'])
data['total_used_bytes'] = df['stats']['total_used_bytes']
data['total_bytes'] = df['stats']['total_bytes']
data['total_avail_bytes'] = df['stats']['total_avail_bytes']
wr_ops = 0
rd_ops = 0
wr_bytes = 0
rd_bytes = 0
for pool in df['pools']:
wr_ops += pool['stats']['wr']
rd_ops += pool['stats']['rd']
wr_bytes += pool['stats']['wr_bytes']
rd_bytes += pool['stats']['rd_bytes']
data['[{0},rd_bytes]'.format(
pool['name'])] = pool['stats']['rd_bytes']
data['[{0},wr_bytes]'.format(
pool['name'])] = pool['stats']['wr_bytes']
data['[{0},rd_ops]'.format(pool['name'])] = pool['stats']['rd']
data['[{0},wr_ops]'.format(pool['name'])] = pool['stats']['wr']
data['[{0},bytes_used]'.format(
pool['name'])] = pool['stats']['bytes_used']
data['[{0},stored_raw]'.format(
pool['name'])] = pool['stats']['stored_raw']
data['[{0},percent_used]'.format(
pool['name'])] = pool['stats']['percent_used'] * 100
data['wr_ops'] = wr_ops
data['rd_ops'] = rd_ops
data['wr_bytes'] = wr_bytes
data['rd_bytes'] = rd_bytes
osd_map = self.get('osd_map')
data['num_osd'] = len(osd_map['osds'])
data['osd_nearfull_ratio'] = osd_map['nearfull_ratio']
data['osd_full_ratio'] = osd_map['full_ratio']
data['osd_backfillfull_ratio'] = osd_map['backfillfull_ratio']
data['num_pg_temp'] = len(osd_map['pg_temp'])
num_up = 0
num_in = 0
for osd in osd_map['osds']:
data['[osd.{0},up]'.format(int(osd['osd']))] = osd['up']
if osd['up'] == 1:
num_up += 1
data['[osd.{0},in]'.format(int(osd['osd']))] = osd['in']
if osd['in'] == 1:
num_in += 1
data['num_osd_up'] = num_up
data['num_osd_in'] = num_in
osd_fill = list()
osd_pgs = list()
osd_apply_latency_ns = list()
osd_commit_latency_ns = list()
osd_stats = self.get('osd_stats')
for osd in osd_stats['osd_stats']:
try:
osd_fill.append(
(float(osd['kb_used']) / float(osd['kb'])) * 100)
data['[osd.{0},osd_fill]'.format(
osd['osd'])] = (float(osd['kb_used']) /
float(osd['kb'])) * 100
except ZeroDivisionError:
continue
osd_pgs.append(osd['num_pgs'])
osd_apply_latency_ns.append(osd['perf_stat']['apply_latency_ns'])
osd_commit_latency_ns.append(osd['perf_stat']['commit_latency_ns'])
data['[osd.{0},num_pgs]'.format(osd['osd'])] = osd['num_pgs']
data['[osd.{0},osd_latency_apply]'.format(
osd['osd']
)] = osd['perf_stat']['apply_latency_ns'] / 1000000.0 # ns -> ms
data['[osd.{0},osd_latency_commit]'.format(
osd['osd']
)] = osd['perf_stat']['commit_latency_ns'] / 1000000.0 # ns -> ms
try:
data['osd_max_fill'] = max(osd_fill)
data['osd_min_fill'] = min(osd_fill)
data['osd_avg_fill'] = avg(osd_fill)
data['osd_max_pgs'] = max(osd_pgs)
data['osd_min_pgs'] = min(osd_pgs)
data['osd_avg_pgs'] = avg(osd_pgs)
except ValueError:
pass
try:
data['osd_latency_apply_max'] = max(
osd_apply_latency_ns) / 1000000.0 # ns -> ms
data['osd_latency_apply_min'] = min(
osd_apply_latency_ns) / 1000000.0 # ns -> ms
data['osd_latency_apply_avg'] = avg(
osd_apply_latency_ns) / 1000000.0 # ns -> ms
data['osd_latency_commit_max'] = max(
osd_commit_latency_ns) / 1000000.0 # ns -> ms
data['osd_latency_commit_min'] = min(
osd_commit_latency_ns) / 1000000.0 # ns -> ms
data['osd_latency_commit_avg'] = avg(
osd_commit_latency_ns) / 1000000.0 # ns -> ms
except ValueError:
pass
data.update(self.get_pg_stats())
return data
def send(self, data: Mapping[str, Union[int, float, str]]) -> bool:
identifier = cast(Optional[str], self.config['identifier'])
if identifier is None or len(identifier) == 0:
identifier = 'ceph-{0}'.format(self.fsid)
if not self.config['zabbix_host'] or not self._zabbix_hosts:
self.log.error('Zabbix server not set, please configure using: '
'ceph zabbix config-set zabbix_host <zabbix_host>')
self.set_health_checks({
'MGR_ZABBIX_NO_SERVER': {
'severity': 'warning',
'summary': 'No Zabbix server configured',
'detail':
['Configuration value zabbix_host not configured']
}
})
return False
result = True
for server in self._zabbix_hosts:
self.log.info(
'Sending data to Zabbix server %s, port %s as host/identifier %s',
server['zabbix_host'], server['zabbix_port'], identifier)
self.log.debug(data)
try:
zabbix = ZabbixSender(cast(str, self.config['zabbix_sender']),
cast(str, server['zabbix_host']),
cast(int, server['zabbix_port']),
self.log)
zabbix.send(identifier, data)
except Exception as exc:
self.log.exception('Failed to send.')
self.set_health_checks({
'MGR_ZABBIX_SEND_FAILED': {
'severity': 'warning',
'summary': 'Failed to send data to Zabbix',
'detail': [str(exc)]
}
})
result = False
self.set_health_checks(dict())
return result
def discovery(self) -> bool:
osd_map = self.get('osd_map')
osd_map_crush = self.get('osd_map_crush')
# Discovering ceph pools
pool_discovery = {
pool['pool_name']: step['item_name']
for pool in osd_map['pools'] for rule in osd_map_crush['rules']
if rule['rule_id'] == pool['crush_rule'] for step in rule['steps']
if step['op'] == "take"
}
pools_discovery_data = {
"data": [{
"{#POOL}": pool,
"{#CRUSH_RULE}": rule
} for pool, rule in pool_discovery.items()]
}
# Discovering OSDs
# Getting hosts for found crush rules
osd_roots = {
step['item_name']: [item['id'] for item in root_bucket['items']]
for rule in osd_map_crush['rules'] for step in rule['steps']
if step['op'] == "take" for root_bucket in osd_map_crush['buckets']
if root_bucket['id'] == step['item']
}
# Getting osds for hosts with map to crush_rule
osd_discovery = {
item['id']: crush_rule
for crush_rule, roots in osd_roots.items() for root in roots
for bucket in osd_map_crush['buckets'] if bucket['id'] == root
for item in bucket['items']
}
osd_discovery_data = {
"data": [{
"{#OSD}": osd,
"{#CRUSH_RULE}": rule
} for osd, rule in osd_discovery.items()]
}
# Preparing recieved data for sending
data = {
"zabbix.pool.discovery": json.dumps(pools_discovery_data),
"zabbix.osd.discovery": json.dumps(osd_discovery_data)
}
return bool(self.send(data))
@CLIReadCommand('zabbix config-show')
def config_show(self) -> Tuple[int, str, str]:
return 0, json.dumps(self.config, indent=4, sort_keys=True), ''
@CLIWriteCommand('zabbix config-set')
def config_set(self, key: str, value: str) -> Tuple[int, str, str]:
if not value:
return -errno.EINVAL, '', 'Value should not be empty or None'
self.log.debug('Setting configuration option %s to %s', key, value)
if self.set_config_option(key, value):
self.set_module_option(key, value)
if key == 'zabbix_host' or key == 'zabbix_port':
self._parse_zabbix_hosts()
return 0, 'Configuration option {0} updated'.format(key), ''
return 1,\
'Failed to update configuration option {0}'.format(key), ''
@CLIReadCommand('zabbix send')
def do_send(self) -> Tuple[int, str, str]:
data = self.get_data()
if self.send(data):
return 0, 'Sending data to Zabbix', ''
return 1, 'Failed to send data to Zabbix', ''
@CLIReadCommand('zabbix discovery')
def do_discovery(self) -> Tuple[int, str, str]:
if self.discovery():
return 0, 'Sending discovery data to Zabbix', ''
return 1, 'Failed to send discovery data to Zabbix', ''
def shutdown(self) -> None:
self.log.info('Stopping zabbix')
self.run = False
self.event.set()
def serve(self) -> None:
self.log.info('Zabbix module starting up')
self.run = True
self.init_module_config()
discovery_interval = self.config['discovery_interval']
# We are sending discovery once plugin is loaded
discovery_counter = cast(int, discovery_interval)
while self.run:
self.log.debug('Waking up for new iteration')
if discovery_counter == discovery_interval:
try:
self.discovery()
except Exception:
# Shouldn't happen, but let's log it and retry next interval,
# rather than dying completely.
self.log.exception("Unexpected error during discovery():")
finally:
discovery_counter = 0
try:
data = self.get_data()
self.send(data)
except Exception:
# Shouldn't happen, but let's log it and retry next interval,
# rather than dying completely.
self.log.exception("Unexpected error during send():")
interval = cast(float, self.config['interval'])
self.log.debug('Sleeping for %d seconds', interval)
discovery_counter += 1
self.event.wait(interval)
def self_test(self) -> None:
data = self.get_data()
if data['overall_status'] not in self.ceph_health_mapping:
raise RuntimeError('No valid overall_status found in data')
int(data['overall_status_int'])
if data['num_mon'] < 1:
raise RuntimeError('num_mon is smaller than 1')
class Module(MgrModule):
COMMANDS = [{
"cmd": "progress",
"desc": "Show progress of recovery operations",
"perm": "r"
}, {
"cmd": "progress json",
"desc": "Show machine readable progress information",
"perm": "r"
}, {
"cmd": "progress clear",
"desc": "Reset progress tracking",
"perm": "rw"
}, {
"cmd": "progress on",
"desc": "Enable progress tracking",
"perm": "rw"
}, {
"cmd": "progress off",
"desc": "Disable progress tracking",
"perm": "rw"
}]
MODULE_OPTIONS = [
Option('max_completed_events',
default=50,
type='int',
desc='number of past completed events to remember',
runtime=True),
Option('persist_interval',
default=5,
type='secs',
desc='how frequently to persist completed events',
runtime=True),
Option(
'enabled',
default=True,
type='bool',
)
]
def __init__(self, *args, **kwargs):
super(Module, self).__init__(*args, **kwargs)
self._events = {
} # type: Dict[str, Union[RemoteEvent, PgRecoveryEvent, GlobalRecoveryEvent]]
self._completed_events = [] # type: List[GhostEvent]
self._old_osd_map = None # type: Optional[OSDMap]
self._ready = threading.Event()
self._shutdown = threading.Event()
self._latest_osdmap = None # type: Optional[OSDMap]
self._dirty = False
global _module
_module = self
# only for mypy
if TYPE_CHECKING:
self.max_completed_events = 0
self.persist_interval = 0
self.enabled = True
def config_notify(self):
for opt in self.MODULE_OPTIONS:
setattr(self, opt['name'], self.get_module_option(opt['name']))
self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name']))
def _osd_in_out(self, old_map, old_dump, new_map, osd_id, marked):
# type: (OSDMap, Dict, OSDMap, str, str) -> None
# A function that will create or complete an event when an
# OSD is marked in or out according to the affected PGs
affected_pgs = []
unmoved_pgs = []
for pool in old_dump['pools']:
pool_id = pool['pool'] # type: str
for ps in range(0, pool['pg_num']):
# Was this OSD affected by the OSD coming in/out?
# Compare old and new osds using
# data from the json dump
old_up_acting = old_map.pg_to_up_acting_osds(pool['pool'], ps)
old_osds = set(old_up_acting['acting'])
new_up_acting = new_map.pg_to_up_acting_osds(pool['pool'], ps)
new_osds = set(new_up_acting['acting'])
# Check the osd_id being in the acting set for both old
# and new maps to cover both out and in cases
was_on_out_or_in_osd = osd_id in old_osds or osd_id in new_osds
if not was_on_out_or_in_osd:
continue
self.log.debug("pool_id, ps = {0}, {1}".format(pool_id, ps))
self.log.debug("old_up_acting: {0}".format(
json.dumps(old_up_acting, indent=4, sort_keys=True)))
# Has this OSD been assigned a new location?
# (it might not be if there is no suitable place to move
# after an OSD is marked in/out)
if marked == "in":
is_relocated = len(old_osds - new_osds) > 0
else:
is_relocated = len(new_osds - old_osds) > 0
self.log.debug("new_up_acting: {0}".format(
json.dumps(new_up_acting, indent=4, sort_keys=True)))
if was_on_out_or_in_osd and is_relocated:
# This PG is now in motion, track its progress
affected_pgs.append(PgId(pool_id, ps))
elif not is_relocated:
# This PG didn't get a new location, we'll log it
unmoved_pgs.append(PgId(pool_id, ps))
# In the case that we ignored some PGs, log the reason why (we may
# not end up creating a progress event)
if len(unmoved_pgs):
self.log.warning(
"{0} PGs were on osd.{1}, but didn't get new locations".format(
len(unmoved_pgs), osd_id))
self.log.warning("{0} PGs affected by osd.{1} being marked {2}".format(
len(affected_pgs), osd_id, marked))
# In the case of the osd coming back in, we might need to cancel
# previous recovery event for that osd
if marked == "in":
for ev_id in list(self._events):
ev = self._events[ev_id]
if isinstance(ev, PgRecoveryEvent) and osd_id in ev.which_osds:
self.log.info(
"osd.{0} came back in, cancelling event".format(
osd_id))
self._complete(ev)
if len(affected_pgs) > 0:
r_ev = PgRecoveryEvent(
"Rebalancing after osd.{0} marked {1}".format(osd_id, marked),
refs=[("osd", osd_id)],
which_pgs=affected_pgs,
which_osds=[osd_id],
start_epoch=self.get_osdmap().get_epoch(),
add_to_ceph_s=False)
r_ev.pg_update(self.get("pg_stats"), self.get("pg_ready"),
self.log)
self._events[r_ev.id] = r_ev
def _osdmap_changed(self, old_osdmap, new_osdmap):
# type: (OSDMap, OSDMap) -> None
old_dump = old_osdmap.dump()
new_dump = new_osdmap.dump()
old_osds = dict([(o['osd'], o) for o in old_dump['osds']])
for osd in new_dump['osds']:
osd_id = osd['osd']
new_weight = osd['in']
if osd_id in old_osds:
old_weight = old_osds[osd_id]['in']
if new_weight == 0.0 and old_weight > new_weight:
self.log.warning("osd.{0} marked out".format(osd_id))
self._osd_in_out(old_osdmap, old_dump, new_osdmap, osd_id,
"out")
elif new_weight >= 1.0 and old_weight == 0.0:
# Only consider weight>=1.0 as "in" to avoid spawning
# individual recovery events on every adjustment
# in a gradual weight-in
self.log.warning("osd.{0} marked in".format(osd_id))
self._osd_in_out(old_osdmap, old_dump, new_osdmap, osd_id,
"in")
def _pg_state_changed(self, pg_dump):
# This function both constructs and updates
# the global recovery event if one of the
# PGs is not at active+clean state
pgs = pg_dump['pg_stats']
total_pg_num = len(pgs)
active_clean_num = 0
for pg in pgs:
state = pg['state']
states = state.split("+")
if "active" in states and "clean" in states:
active_clean_num += 1
try:
# There might be a case where there is no pg_num
progress = float(active_clean_num) / total_pg_num
except ZeroDivisionError:
return
if progress < 1.0:
ev = GlobalRecoveryEvent("Global Recovery Event",
refs=[("global", "")],
add_to_ceph_s=True,
start_epoch=self.get_osdmap().get_epoch(),
active_clean_num=active_clean_num)
ev.global_event_update_progress(pg_dump)
self._events[ev.id] = ev
def notify(self, notify_type, notify_data):
self._ready.wait()
if not self.enabled:
return
if notify_type == "osd_map":
old_osdmap = self._latest_osdmap
self._latest_osdmap = self.get_osdmap()
assert old_osdmap
assert self._latest_osdmap
self.log.info("Processing OSDMap change {0}..{1}".format(
old_osdmap.get_epoch(), self._latest_osdmap.get_epoch()))
self._osdmap_changed(old_osdmap, self._latest_osdmap)
elif notify_type == "pg_summary":
# if there are no events we will skip this here to avoid
# expensive get calls
if len(self._events) == 0:
return
global_event = False
data = self.get("pg_stats")
ready = self.get("pg_ready")
for ev_id in list(self._events):
ev = self._events[ev_id]
# Check for types of events
# we have to update
if isinstance(ev, PgRecoveryEvent):
ev.pg_update(data, ready, self.log)
self.maybe_complete(ev)
elif isinstance(ev, GlobalRecoveryEvent):
global_event = True
ev.global_event_update_progress(data)
self.maybe_complete(ev)
if not global_event:
# If there is no global event
# we create one
self._pg_state_changed(data)
def maybe_complete(self, event):
# type: (Event) -> None
if event.progress >= 1.0:
self._complete(event)
def _save(self):
self.log.info("Writing back {0} completed events".format(
len(self._completed_events)))
# TODO: bound the number we store.
encoded = json.dumps({
"events": [ev.to_json() for ev in self._completed_events],
"version":
ENCODING_VERSION,
"compat_version":
ENCODING_VERSION
})
self.set_store("completed", encoded)
def _load(self):
stored = self.get_store("completed")
if stored is None:
self.log.info("No stored events to load")
return
decoded = json.loads(stored)
if decoded['compat_version'] > ENCODING_VERSION:
raise RuntimeError("Cannot decode version {0}".format(
decoded['compat_version']))
if decoded['compat_version'] < ENCODING_VERSION:
# we need to add the "started_at" and "finished_at" attributes to the events
for ev in decoded['events']:
ev['started_at'] = None
ev['finished_at'] = None
for ev in decoded['events']:
self._completed_events.append(
GhostEvent(ev['id'], ev['message'], ev['refs'],
ev['started_at'], ev['finished_at'],
ev.get('failed', False), ev.get('failure_message')))
self._prune_completed_events()
def _prune_completed_events(self):
length = len(self._completed_events)
if length > self.max_completed_events:
self._completed_events = self._completed_events[
length - self.max_completed_events:length]
self._dirty = True
def serve(self):
self.config_notify()
self.clear_all_progress_events()
self.log.info("Loading...")
self._load()
self.log.info("Loaded {0} historic events".format(
self._completed_events))
self._latest_osdmap = self.get_osdmap()
self.log.info("Loaded OSDMap, ready.")
self._ready.set()
while not self._shutdown.is_set():
# Lazy periodic write back of completed events
if self._dirty:
self._save()
self._dirty = False
self._shutdown.wait(timeout=self.persist_interval)
self._shutdown.wait()
def shutdown(self):
self._shutdown.set()
self.clear_all_progress_events()
def update(self,
ev_id,
ev_msg,
ev_progress,
refs=None,
add_to_ceph_s=False):
# type: (str, str, float, Optional[list], bool) -> None
"""
For calling from other mgr modules
"""
if not self.enabled:
return
if refs is None:
refs = []
try:
ev = self._events[ev_id]
assert isinstance(ev, RemoteEvent)
except KeyError:
ev = RemoteEvent(ev_id, ev_msg, refs, add_to_ceph_s)
self._events[ev_id] = ev
self.log.info("update: starting ev {0} ({1})".format(
ev_id, ev_msg))
else:
self.log.debug("update: {0} on {1}".format(ev_progress, ev_msg))
ev.set_progress(ev_progress)
ev.set_message(ev_msg)
def _complete(self, ev):
# type: (Event) -> None
duration = (time.time() - ev.started_at)
self.log.info("Completed event {0} ({1}) in {2} seconds".format(
ev.id, ev.message, int(round(duration))))
self.complete_progress_event(ev.id)
self._completed_events.append(
GhostEvent(ev.id,
ev.message,
ev.refs,
ev.add_to_ceph_s,
ev.started_at,
failed=ev.failed,
failure_message=ev.failure_message))
assert ev.id
del self._events[ev.id]
self._prune_completed_events()
self._dirty = True
def complete(self, ev_id):
"""
For calling from other mgr modules
"""
if not self.enabled:
return
try:
ev = self._events[ev_id]
assert isinstance(ev, RemoteEvent)
ev.set_progress(1.0)
self.log.info("complete: finished ev {0} ({1})".format(
ev_id, ev.message))
self._complete(ev)
except KeyError:
self.log.warning("complete: ev {0} does not exist".format(ev_id))
pass
def fail(self, ev_id, message):
"""
For calling from other mgr modules to mark an event as failed (and
complete)
"""
try:
ev = self._events[ev_id]
assert isinstance(ev, RemoteEvent)
ev.set_failed(message)
self.log.info("fail: finished ev {0} ({1}): {2}".format(
ev_id, ev.message, message))
self._complete(ev)
except KeyError:
self.log.warning("fail: ev {0} does not exist".format(ev_id))
def on(self):
self.set_module_option('enabled', "true")
def off(self):
self.set_module_option('enabled', "false")
def _handle_ls(self):
if len(self._events) or len(self._completed_events):
out = ""
chrono_order = sorted(self._events.values(),
key=lambda x: x.started_at,
reverse=True)
for ev in chrono_order:
out += ev.twoline_progress()
out += "\n"
if len(self._completed_events):
# TODO: limit number of completed events to show
out += "\n"
for ghost_ev in self._completed_events:
out += "[{0}]: {1}\n".format(
"Complete" if not ghost_ev.failed else "Failed",
ghost_ev.twoline_progress())
return 0, out, ""
else:
return 0, "", "Nothing in progress"
def _json(self):
return {
'events': [ev.to_json() for ev in self._events.values()],
'completed': [ev.to_json() for ev in self._completed_events]
}
def clear(self):
self._events = {}
self._completed_events = []
self._dirty = True
self._save()
self.clear_all_progress_events()
def _handle_clear(self):
self.clear()
return 0, "", ""
def handle_command(self, _, cmd):
if cmd['prefix'] == "progress":
return self._handle_ls()
elif cmd['prefix'] == "progress clear":
# The clear command isn't usually needed - it's to enable
# the admin to "kick" this module if it seems to have done
# something wrong (e.g. we have a bug causing a progress event
# that never finishes)
return self._handle_clear()
elif cmd['prefix'] == "progress json":
return 0, json.dumps(self._json(), indent=4, sort_keys=True), ""
elif cmd['prefix'] == "progress on":
if self.enabled:
return 0, "", "progress already enabled!"
self.on()
return 0, "", "progress enabled"
elif cmd['prefix'] == "progress off":
if not self.enabled:
return 0, "", "progress already disabled!"
self.off()
self.clear()
return 0, "", "progress disabled"
else:
raise NotImplementedError(cmd['prefix'])
class RookOrchestrator(MgrModule, orchestrator.Orchestrator):
"""
Writes are a two-phase thing, firstly sending
the write to the k8s API (fast) and then waiting
for the corresponding change to appear in the
Ceph cluster (slow)
Right now, we are calling the k8s API synchronously.
"""
MODULE_OPTIONS: List[Option] = [
# TODO: configure k8s API addr instead of assuming local
Option(
'storage_class',
type='str',
default='local',
desc='storage class name for LSO-discovered PVs',
),
]
@staticmethod
def can_run() -> Tuple[bool, str]:
if not kubernetes_imported:
return False, "`kubernetes` python module not found"
if not RookEnv().api_version_match():
return False, "Rook version unsupported."
return True, ''
def available(self) -> Tuple[bool, str, Dict[str, Any]]:
if not kubernetes_imported:
return False, "`kubernetes` python module not found", {}
elif not self._rook_env.has_namespace():
return False, "ceph-mgr not running in Rook cluster", {}
try:
self.k8s.list_namespaced_pod(self._rook_env.namespace)
except ApiException as e:
return False, "Cannot reach Kubernetes API: {}".format(e), {}
else:
return True, "", {}
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(RookOrchestrator, self).__init__(*args, **kwargs)
self._initialized = threading.Event()
self._k8s_CoreV1_api: Optional[client.CoreV1Api] = None
self._k8s_BatchV1_api: Optional[client.BatchV1Api] = None
self._k8s_CustomObjects_api: Optional[client.CustomObjectsApi] = None
self._k8s_StorageV1_api: Optional[client.StorageV1Api] = None
self._rook_cluster: Optional[RookCluster] = None
self._rook_env = RookEnv()
self.storage_class = self.get_module_option('storage_class')
self._shutdown = threading.Event()
def config_notify(self) -> None:
"""
This method is called whenever one of our config options is changed.
TODO: this method should be moved into mgr_module.py
"""
for opt in self.MODULE_OPTIONS:
setattr(
self,
opt['name'], # type: ignore
self.get_module_option(opt['name'])) # type: ignore
self.log.debug(' mgr option %s = %s', opt['name'],
getattr(self, opt['name'])) # type: ignore
assert isinstance(self.storage_class, str)
self.rook_cluster.storage_class = self.storage_class
def shutdown(self) -> None:
self._shutdown.set()
@property
def k8s(self):
# type: () -> client.CoreV1Api
self._initialized.wait()
assert self._k8s_CoreV1_api is not None
return self._k8s_CoreV1_api
@property
def rook_cluster(self):
# type: () -> RookCluster
self._initialized.wait()
assert self._rook_cluster is not None
return self._rook_cluster
def serve(self) -> None:
# For deployed clusters, we should always be running inside
# a Rook cluster. For development convenience, also support
# running outside (reading ~/.kube config)
if self._rook_env.has_namespace():
config.load_incluster_config()
else:
self.log.warning("DEVELOPMENT ONLY: Reading kube config from ~")
config.load_kube_config()
# So that I can do port forwarding from my workstation - jcsp
from kubernetes.client import configuration
configuration.verify_ssl = False
self._k8s_CoreV1_api = client.CoreV1Api()
self._k8s_BatchV1_api = client.BatchV1Api()
self._k8s_CustomObjects_api = client.CustomObjectsApi()
self._k8s_StorageV1_api = client.StorageV1Api()
try:
# XXX mystery hack -- I need to do an API call from
# this context, or subsequent API usage from handle_command
# fails with SSLError('bad handshake'). Suspect some kind of
# thread context setup in SSL lib?
self._k8s_CoreV1_api.list_namespaced_pod(self._rook_env.namespace)
except ApiException:
# Ignore here to make self.available() fail with a proper error message
pass
assert isinstance(self.storage_class, str)
self._rook_cluster = RookCluster(self._k8s_CoreV1_api,
self._k8s_BatchV1_api,
self._k8s_CustomObjects_api,
self._k8s_StorageV1_api,
self._rook_env, self.storage_class)
self._initialized.set()
while not self._shutdown.is_set():
self._shutdown.wait(5)
@handle_orch_error
def get_inventory(
self,
host_filter: Optional[orchestrator.InventoryFilter] = None,
refresh: bool = False) -> List[orchestrator.InventoryHost]:
host_list = None
if host_filter and host_filter.hosts:
# Explicit host list
host_list = host_filter.hosts
elif host_filter and host_filter.labels:
# TODO: query k8s API to resolve to host list, and pass
# it into RookCluster.get_discovered_devices
raise NotImplementedError()
discovered_devs = self.rook_cluster.get_discovered_devices(host_list)
result = []
for host_name, host_devs in discovered_devs.items():
devs = []
for d in host_devs:
devs.append(d)
result.append(
orchestrator.InventoryHost(host_name, inventory.Devices(devs)))
return result
@handle_orch_error
def get_hosts(self):
# type: () -> List[orchestrator.HostSpec]
return [
orchestrator.HostSpec(n)
for n in self.rook_cluster.get_node_names()
]
@handle_orch_error
def describe_service(
self,
service_type: Optional[str] = None,
service_name: Optional[str] = None,
refresh: bool = False) -> List[orchestrator.ServiceDescription]:
now = datetime_now()
# CephCluster
cl = self.rook_cluster.rook_api_get("cephclusters/{0}".format(
self.rook_cluster.rook_env.cluster_name))
self.log.debug('CephCluster %s' % cl)
image_name = cl['spec'].get('cephVersion', {}).get('image', None)
num_nodes = len(self.rook_cluster.get_node_names())
spec = {}
if service_type == 'mon' or service_type is None:
spec['mon'] = orchestrator.ServiceDescription(
spec=ServiceSpec(
'mon',
placement=PlacementSpec(count=cl['spec'].get(
'mon', {}).get('count', 1), ),
),
size=cl['spec'].get('mon', {}).get('count', 1),
container_image_name=image_name,
last_refresh=now,
)
if service_type == 'mgr' or service_type is None:
spec['mgr'] = orchestrator.ServiceDescription(
spec=ServiceSpec(
'mgr',
placement=PlacementSpec.from_string('count:1'),
),
size=1,
container_image_name=image_name,
last_refresh=now,
)
if not cl['spec'].get('crashCollector', {}).get('disable', False):
spec['crash'] = orchestrator.ServiceDescription(
spec=ServiceSpec(
'crash',
placement=PlacementSpec.from_string('*'),
),
size=num_nodes,
container_image_name=image_name,
last_refresh=now,
)
if service_type == 'mds' or service_type is None:
# CephFilesystems
all_fs = self.rook_cluster.rook_api_get("cephfilesystems/")
self.log.debug('CephFilesystems %s' % all_fs)
for fs in all_fs.get('items', []):
svc = 'mds.' + fs['metadata']['name']
if svc in spec:
continue
# FIXME: we are conflating active (+ standby) with count
active = fs['spec'].get('metadataServer',
{}).get('activeCount', 1)
total_mds = active
if fs['spec'].get('metadataServer',
{}).get('activeStandby', False):
total_mds = active * 2
spec[svc] = orchestrator.ServiceDescription(
spec=ServiceSpec(
service_type='mds',
service_id=fs['metadata']['name'],
placement=PlacementSpec(count=active),
),
size=total_mds,
container_image_name=image_name,
last_refresh=now,
)
if service_type == 'rgw' or service_type is None:
# CephObjectstores
all_zones = self.rook_cluster.rook_api_get("cephobjectstores/")
self.log.debug('CephObjectstores %s' % all_zones)
for zone in all_zones.get('items', []):
rgw_realm = zone['metadata']['name']
rgw_zone = rgw_realm
svc = 'rgw.' + rgw_realm + '.' + rgw_zone
if svc in spec:
continue
active = zone['spec']['gateway']['instances']
if 'securePort' in zone['spec']['gateway']:
ssl = True
port = zone['spec']['gateway']['securePort']
else:
ssl = False
port = zone['spec']['gateway']['port'] or 80
spec[svc] = orchestrator.ServiceDescription(
spec=RGWSpec(
service_id=rgw_realm + '.' + rgw_zone,
rgw_realm=rgw_realm,
rgw_zone=rgw_zone,
ssl=ssl,
rgw_frontend_port=port,
placement=PlacementSpec(count=active),
),
size=active,
container_image_name=image_name,
last_refresh=now,
)
if service_type == 'nfs' or service_type is None:
# CephNFSes
all_nfs = self.rook_cluster.rook_api_get("cephnfses/")
self.log.warning('CephNFS %s' % all_nfs)
for nfs in all_nfs.get('items', []):
nfs_name = nfs['metadata']['name']
svc = 'nfs.' + nfs_name
if svc in spec:
continue
active = nfs['spec'].get('server', {}).get('active')
spec[svc] = orchestrator.ServiceDescription(
spec=NFSServiceSpec(
service_id=nfs_name,
placement=PlacementSpec(count=active),
),
size=active,
last_refresh=now,
)
for dd in self._list_daemons():
if dd.service_name() not in spec:
continue
service = spec[dd.service_name()]
service.running += 1
if not service.container_image_id:
service.container_image_id = dd.container_image_id
if not service.container_image_name:
service.container_image_name = dd.container_image_name
if service.last_refresh is None or not dd.last_refresh or dd.last_refresh < service.last_refresh:
service.last_refresh = dd.last_refresh
if service.created is None or dd.created is None or dd.created < service.created:
service.created = dd.created
return [v for k, v in spec.items()]
@handle_orch_error
def list_daemons(
self,
service_name: Optional[str] = None,
daemon_type: Optional[str] = None,
daemon_id: Optional[str] = None,
host: Optional[str] = None,
refresh: bool = False) -> List[orchestrator.DaemonDescription]:
return self._list_daemons(service_name=service_name,
daemon_type=daemon_type,
daemon_id=daemon_id,
host=host,
refresh=refresh)
def _list_daemons(
self,
service_name: Optional[str] = None,
daemon_type: Optional[str] = None,
daemon_id: Optional[str] = None,
host: Optional[str] = None,
refresh: bool = False) -> List[orchestrator.DaemonDescription]:
pods = self.rook_cluster.describe_pods(daemon_type, daemon_id, host)
self.log.debug('pods %s' % pods)
result = []
for p in pods:
sd = orchestrator.DaemonDescription()
sd.hostname = p['hostname']
sd.daemon_type = p['labels']['app'].replace('rook-ceph-', '')
status = {
'Pending': orchestrator.DaemonDescriptionStatus.error,
'Running': orchestrator.DaemonDescriptionStatus.running,
'Succeeded': orchestrator.DaemonDescriptionStatus.stopped,
'Failed': orchestrator.DaemonDescriptionStatus.error,
'Unknown': orchestrator.DaemonDescriptionStatus.error,
}[p['phase']]
sd.status = status
sd.status_desc = p['phase']
if 'ceph_daemon_id' in p['labels']:
sd.daemon_id = p['labels']['ceph_daemon_id']
elif 'ceph-osd-id' in p['labels']:
sd.daemon_id = p['labels']['ceph-osd-id']
else:
# Unknown type -- skip it
continue
if service_name is not None and service_name != sd.service_name():
continue
sd.container_image_name = p['container_image_name']
sd.container_image_id = p['container_image_id']
sd.created = p['created']
sd.last_configured = p['created']
sd.last_deployed = p['created']
sd.started = p['started']
sd.last_refresh = p['refreshed']
result.append(sd)
return result
@handle_orch_error
def remove_service(self, service_name: str) -> str:
service_type, service_name = service_name.split('.', 1)
if service_type == 'mds':
return self.rook_cluster.rm_service('cephfilesystems',
service_name)
elif service_type == 'rgw':
return self.rook_cluster.rm_service('cephobjectstores',
service_name)
elif service_type == 'nfs':
return self.rook_cluster.rm_service('cephnfses', service_name)
else:
raise orchestrator.OrchestratorError(
f'Service type {service_type} not supported')
@handle_orch_error
def apply_mon(self, spec):
# type: (ServiceSpec) -> str
if spec.placement.hosts or spec.placement.label:
raise RuntimeError("Host list or label is not supported by rook.")
return self.rook_cluster.update_mon_count(spec.placement.count)
@handle_orch_error
def apply_mds(self, spec):
# type: (ServiceSpec) -> str
return self.rook_cluster.apply_filesystem(spec)
@handle_orch_error
def apply_rgw(self, spec):
# type: (RGWSpec) -> str
return self.rook_cluster.apply_objectstore(spec)
@handle_orch_error
def apply_nfs(self, spec):
# type: (NFSServiceSpec) -> str
return self.rook_cluster.apply_nfsgw(spec)
@handle_orch_error
def remove_daemons(self, names: List[str]) -> List[str]:
return self.rook_cluster.remove_pods(names)
def apply_drivegroups(
self, specs: List[DriveGroupSpec]) -> OrchResult[List[str]]:
result_list = []
all_hosts = raise_if_exception(self.get_hosts())
for drive_group in specs:
matching_hosts = drive_group.placement.filter_matching_hosts(
lambda label=None, as_hostspec=None: all_hosts)
if not self.rook_cluster.node_exists(matching_hosts[0]):
raise RuntimeError("Node '{0}' is not in the Kubernetes "
"cluster".format(matching_hosts))
# Validate whether cluster CRD can accept individual OSD
# creations (i.e. not useAllDevices)
if not self.rook_cluster.can_create_osd():
raise RuntimeError("Rook cluster configuration does not "
"support OSD creation.")
result_list.append(
self.rook_cluster.add_osds(drive_group, matching_hosts))
return OrchResult(result_list)
"""
@handle_orch_error
def create_osds(self, drive_group):
# type: (DriveGroupSpec) -> str
# Creates OSDs from a drive group specification.
# $: ceph orch osd create -i <dg.file>
# The drivegroup file must only contain one spec at a time.
#
targets = [] # type: List[str]
if drive_group.data_devices and drive_group.data_devices.paths:
targets += [d.path for d in drive_group.data_devices.paths]
if drive_group.data_directories:
targets += drive_group.data_directories
all_hosts = raise_if_exception(self.get_hosts())
matching_hosts = drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: all_hosts)
assert len(matching_hosts) == 1
if not self.rook_cluster.node_exists(matching_hosts[0]):
raise RuntimeError("Node '{0}' is not in the Kubernetes "
"cluster".format(matching_hosts))
# Validate whether cluster CRD can accept individual OSD
# creations (i.e. not useAllDevices)
if not self.rook_cluster.can_create_osd():
raise RuntimeError("Rook cluster configuration does not "
"support OSD creation.")
return self.rook_cluster.add_osds(drive_group, matching_hosts)
# TODO: this was the code to update the progress reference:
@handle_orch_error
def has_osds(matching_hosts: List[str]) -> bool:
# Find OSD pods on this host
pod_osd_ids = set()
pods = self.k8s.list_namespaced_pod(self._rook_env.namespace,
label_selector="rook_cluster={},app=rook-ceph-osd".format(self._rook_env.cluster_name),
field_selector="spec.nodeName={0}".format(
matching_hosts[0]
)).items
for p in pods:
pod_osd_ids.add(int(p.metadata.labels['ceph-osd-id']))
self.log.debug('pod_osd_ids={0}'.format(pod_osd_ids))
found = []
osdmap = self.get("osd_map")
for osd in osdmap['osds']:
osd_id = osd['osd']
if osd_id not in pod_osd_ids:
continue
metadata = self.get_metadata('osd', "%s" % osd_id)
if metadata and metadata['devices'] in targets:
found.append(osd_id)
else:
self.log.info("ignoring osd {0} {1}".format(
osd_id, metadata['devices'] if metadata else 'DNE'
))
return found is not None
"""
@handle_orch_error
def blink_device_light(
self, ident_fault: str, on: bool,
locs: List[orchestrator.DeviceLightLoc]) -> List[str]:
return self.rook_cluster.blink_light(ident_fault, on, locs)
class Module(MgrModule, CherryPyConfig):
"""
dashboard module entrypoint
"""
COMMANDS = [
{
'cmd': 'dashboard set-jwt-token-ttl '
'name=seconds,type=CephInt',
'desc': 'Set the JWT token TTL in seconds',
'perm': 'w'
},
{
'cmd': 'dashboard get-jwt-token-ttl',
'desc': 'Get the JWT token TTL in seconds',
'perm': 'r'
},
{
"cmd": "dashboard create-self-signed-cert",
"desc": "Create self signed certificate",
"perm": "w"
},
{
"cmd": "dashboard grafana dashboards update",
"desc": "Push dashboards to Grafana",
"perm": "w",
},
]
COMMANDS.extend(options_command_list())
COMMANDS.extend(SSO_COMMANDS)
PLUGIN_MANAGER.hook.register_commands()
MODULE_OPTIONS = [
Option(name='server_addr', type='str', default=get_default_addr()),
Option(name='server_port', type='int', default=8080),
Option(name='ssl_server_port', type='int', default=8443),
Option(name='jwt_token_ttl', type='int', default=28800),
Option(name='password', type='str', default=''),
Option(name='url_prefix', type='str', default=''),
Option(name='username', type='str', default=''),
Option(name='key_file', type='str', default=''),
Option(name='crt_file', type='str', default=''),
Option(name='ssl', type='bool', default=True),
Option(name='standby_behaviour',
type='str',
default='redirect',
enum_allowed=['redirect', 'error']),
Option(name='standby_error_status_code',
type='int',
default=500,
min=400,
max=599)
]
MODULE_OPTIONS.extend(options_schema_list())
for options in PLUGIN_MANAGER.hook.get_options() or []:
MODULE_OPTIONS.extend(options)
__pool_stats = collections.defaultdict(lambda: collections.defaultdict(
lambda: collections.deque(maxlen=10))) # type: dict
def __init__(self, *args, **kwargs):
super(Module, self).__init__(*args, **kwargs)
CherryPyConfig.__init__(self)
mgr.init(self)
self._stopping = threading.Event()
self.shutdown_event = threading.Event()
self.ACCESS_CTRL_DB = None
self.SSO_DB = None
@classmethod
def can_run(cls):
if cherrypy is None:
return False, "Missing dependency: cherrypy"
if not os.path.exists(cls.get_frontend_path()):
return False, "Frontend assets not found: incomplete build?"
return True, ""
@classmethod
def get_frontend_path(cls):
current_dir = os.path.dirname(os.path.abspath(__file__))
return os.path.join(current_dir, 'frontend/dist')
def serve(self):
AuthManager.initialize()
load_sso_db()
uri = self.await_configuration()
if uri is None:
# We were shut down while waiting
return
# Publish the URI that others may use to access the service we're
# about to start serving
self.set_uri(uri)
mapper, parent_urls = generate_routes(self.url_prefix)
config = {}
for purl in parent_urls:
config[purl] = {'request.dispatch': mapper}
cherrypy.tree.mount(None, config=config)
PLUGIN_MANAGER.hook.setup()
cherrypy.engine.start()
NotificationQueue.start_queue()
TaskManager.init()
logger.info('Engine started.')
update_dashboards = str_to_bool(
self.get_module_option('GRAFANA_UPDATE_DASHBOARDS', 'False'))
if update_dashboards:
logger.info('Starting Grafana dashboard task')
TaskManager.run(
'grafana/dashboards/update',
{},
push_local_dashboards,
kwargs=dict(tries=10, sleep=60),
)
# wait for the shutdown event
self.shutdown_event.wait()
self.shutdown_event.clear()
NotificationQueue.stop()
cherrypy.engine.stop()
logger.info('Engine stopped')
def shutdown(self):
super(Module, self).shutdown()
CherryPyConfig.shutdown(self)
logger.info('Stopping engine...')
self.shutdown_event.set()
@CLIWriteCommand("dashboard set-ssl-certificate",
"name=mgr_id,type=CephString,req=false")
def set_ssl_certificate(self, mgr_id=None, inbuf=None):
if inbuf is None:
return -errno.EINVAL, '',\
'Please specify the certificate file with "-i" option'
if mgr_id is not None:
self.set_store('{}/crt'.format(mgr_id), inbuf)
else:
self.set_store('crt', inbuf)
return 0, 'SSL certificate updated', ''
@CLIWriteCommand("dashboard set-ssl-certificate-key",
"name=mgr_id,type=CephString,req=false")
def set_ssl_certificate_key(self, mgr_id=None, inbuf=None):
if inbuf is None:
return -errno.EINVAL, '',\
'Please specify the certificate key file with "-i" option'
if mgr_id is not None:
self.set_store('{}/key'.format(mgr_id), inbuf)
else:
self.set_store('key', inbuf)
return 0, 'SSL certificate key updated', ''
def handle_command(self, inbuf, cmd):
# pylint: disable=too-many-return-statements
res = handle_option_command(cmd)
if res[0] != -errno.ENOSYS:
return res
res = handle_sso_command(cmd)
if res[0] != -errno.ENOSYS:
return res
if cmd['prefix'] == 'dashboard set-jwt-token-ttl':
self.set_module_option('jwt_token_ttl', str(cmd['seconds']))
return 0, 'JWT token TTL updated', ''
if cmd['prefix'] == 'dashboard get-jwt-token-ttl':
ttl = self.get_module_option('jwt_token_ttl',
JwtManager.JWT_TOKEN_TTL)
return 0, str(ttl), ''
if cmd['prefix'] == 'dashboard create-self-signed-cert':
self.create_self_signed_cert()
return 0, 'Self-signed certificate created', ''
if cmd['prefix'] == 'dashboard grafana dashboards update':
push_local_dashboards()
return 0, 'Grafana dashboards updated', ''
return (-errno.EINVAL, '',
'Command not found \'{0}\''.format(cmd['prefix']))
def create_self_signed_cert(self):
cert, pkey = create_self_signed_cert('IT', 'ceph-dashboard')
self.set_store('crt', cert)
self.set_store('key', pkey)
def notify(self, notify_type, notify_id):
NotificationQueue.new_notification(notify_type, notify_id)
def get_updated_pool_stats(self):
df = self.get('df')
pool_stats = {p['id']: p['stats'] for p in df['pools']}
now = time.time()
for pool_id, stats in pool_stats.items():
for stat_name, stat_val in stats.items():
self.__pool_stats[pool_id][stat_name].append((now, stat_val))
return self.__pool_stats
class Module(MgrModule):
"""
This module is for testing the ceph-mgr python interface from within
a running ceph-mgr daemon.
It implements a sychronous self-test command for calling the functions
in the MgrModule interface one by one, and a background "workload"
command for causing the module to perform some thrashing-type
activities in its serve() thread.
"""
# The test code in qa/ relies on these options existing -- they
# are of course not really used for anything in the module
MODULE_OPTIONS = [
Option(name='testkey'),
Option(name='testlkey'),
Option(name='testnewline'),
Option(name='roption1'),
Option(name='roption2',
type='str',
default='xyz'),
Option(name='rwoption1'),
Option(name='rwoption2',
type='int'),
Option(name='rwoption3',
type='float'),
Option(name='rwoption4',
type='str'),
Option(name='rwoption5',
type='bool'),
Option(name='rwoption6',
type='bool',
default=True),
Option(name='rwoption7',
type='int',
min=1,
max=42),
]
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
self._event = threading.Event()
self._workload: Optional[Workload] = None
self._health: Dict[str, Dict[str, Any]] = {}
self._repl = InteractiveInterpreter(dict(mgr=self))
@CLICommand('mgr self-test python-version', perm='r')
def python_version(self) -> Tuple[int, str, str]:
'''
Query the version of the embedded Python runtime
'''
major = sys.version_info.major
minor = sys.version_info.minor
micro = sys.version_info.micro
return 0, f'{major}.{minor}.{micro}', ''
@CLICommand('mgr self-test run')
def run(self) -> Tuple[int, str, str]:
'''
Run mgr python interface tests
'''
self._self_test()
return 0, '', 'Self-test succeeded'
@CLICommand('mgr self-test background start')
def backgroun_start(self, workload: Workload) -> Tuple[int, str, str]:
'''
Activate a background workload (one of command_spam, throw_exception)
'''
self._workload = workload
self._event.set()
return 0, '', 'Running `{0}` in background'.format(self._workload)
@CLICommand('mgr self-test background stop')
def background_stop(self) -> Tuple[int, str, str]:
'''
Stop background workload if any is running
'''
if self._workload:
was_running = self._workload
self._workload = None
self._event.set()
return 0, '', 'Stopping background workload `{0}`'.format(
was_running)
else:
return 0, '', 'No background workload was running'
@CLICommand('mgr self-test config get')
def config_get(self, key: str) -> Tuple[int, str, str]:
'''
Peek at a configuration value
'''
return 0, str(self.get_module_option(key)), ''
@CLICommand('mgr self-test config get_localized')
def config_get_localized(self, key: str) -> Tuple[int, str, str]:
'''
Peek at a configuration value (localized variant)
'''
return 0, str(self.get_localized_module_option(key)), ''
@CLICommand('mgr self-test remote')
def test_remote(self) -> Tuple[int, str, str]:
'''
Test inter-module calls
'''
self._test_remote_calls()
return 0, '', 'Successfully called'
@CLICommand('mgr self-test module')
def module(self, module: str) -> Tuple[int, str, str]:
'''
Run another module's self_test() method
'''
try:
r = self.remote(module, "self_test")
except RuntimeError as e:
return -1, '', "Test failed: {0}".format(e)
else:
return 0, str(r), "Self-test OK"
@CLICommand('mgr self-test cluster-log')
def do_cluster_log(self,
channel: str,
priority: str,
message: str) -> Tuple[int, str, str]:
'''
Create an audit log record.
'''
priority_map = {
'info': self.ClusterLogPrio.INFO,
'security': self.ClusterLogPrio.SEC,
'warning': self.ClusterLogPrio.WARN,
'error': self.ClusterLogPrio.ERROR
}
self.cluster_log(channel,
priority_map[priority],
message)
return 0, '', 'Successfully called'
@CLICommand('mgr self-test health set')
def health_set(self, checks: str) -> Tuple[int, str, str]:
'''
Set a health check from a JSON-formatted description.
'''
try:
health_check = json.loads(checks)
except Exception as e:
return -1, "", "Failed to decode JSON input: {}".format(e)
try:
for check, info in health_check.items():
self._health[check] = {
"severity": str(info["severity"]),
"summary": str(info["summary"]),
"count": 123,
"detail": [str(m) for m in info["detail"]]
}
except Exception as e:
return -1, "", "Invalid health check format: {}".format(e)
self.set_health_checks(self._health)
return 0, "", ""
@CLICommand('mgr self-test health clear')
def health_clear(self, checks: Optional[List[str]] = None) -> Tuple[int, str, str]:
'''
Clear health checks by name. If no names provided, clear all.
'''
if checks is not None:
for check in checks:
if check in self._health:
del self._health[check]
else:
self._health = dict()
self.set_health_checks(self._health)
return 0, "", ""
@CLICommand('mgr self-test insights_set_now_offset')
def insights_set_now_offset(self, hours: int) -> Tuple[int, str, str]:
'''
Set the now time for the insights module.
'''
self.remote("insights", "testing_set_now_time_offset", hours)
return 0, "", ""
def _self_test(self) -> None:
self.log.info("Running self-test procedure...")
self._self_test_osdmap()
self._self_test_getters()
self._self_test_config()
self._self_test_store()
self._self_test_misc()
self._self_test_perf_counters()
def _self_test_getters(self) -> None:
self.version
self.get_context()
self.get_mgr_id()
# In this function, we will assume that the system is in a steady
# state, i.e. if a server/service appears in one call, it will
# not have gone by the time we call another function referring to it
objects = [
"fs_map",
"osdmap_crush_map_text",
"osd_map",
"config",
"mon_map",
"service_map",
"osd_metadata",
"pg_summary",
"pg_status",
"pg_dump",
"pg_ready",
"df",
"pg_stats",
"pool_stats",
"osd_stats",
"osd_ping_times",
"health",
"mon_status",
"mgr_map"
]
for obj in objects:
assert self.get(obj) is not None
assert self.get("__OBJ_DNE__") is None
servers = self.list_servers()
for server in servers:
self.get_server(server['hostname']) # type: ignore
osdmap = self.get('osd_map')
for o in osdmap['osds']:
osd_id = o['osd']
self.get_metadata("osd", str(osd_id))
self.get_daemon_status("osd", "0")
def _self_test_config(self) -> None:
# This is not a strong test (can't tell if values really
# persisted), it's just for the python interface bit.
self.set_module_option("testkey", "testvalue")
assert self.get_module_option("testkey") == "testvalue"
self.set_localized_module_option("testkey", "foo")
assert self.get_localized_module_option("testkey") == "foo"
# Must return the default value defined in MODULE_OPTIONS.
value = self.get_localized_module_option("rwoption6")
assert isinstance(value, bool)
assert value is True
# Use default value.
assert self.get_module_option("roption1") is None
assert self.get_module_option("roption1", "foobar") == "foobar"
assert self.get_module_option("roption2") == "xyz"
assert self.get_module_option("roption2", "foobar") == "xyz"
# Option type is not defined => return as string.
self.set_module_option("rwoption1", 8080)
value = self.get_module_option("rwoption1")
assert isinstance(value, str)
assert value == "8080"
# Option type is defined => return as integer.
self.set_module_option("rwoption2", 10)
value = self.get_module_option("rwoption2")
assert isinstance(value, int)
assert value == 10
# Option type is defined => return as float.
self.set_module_option("rwoption3", 1.5)
value = self.get_module_option("rwoption3")
assert isinstance(value, float)
assert value == 1.5
# Option type is defined => return as string.
self.set_module_option("rwoption4", "foo")
value = self.get_module_option("rwoption4")
assert isinstance(value, str)
assert value == "foo"
# Option type is defined => return as bool.
self.set_module_option("rwoption5", False)
value = self.get_module_option("rwoption5")
assert isinstance(value, bool)
assert value is False
# Option value range is specified
try:
self.set_module_option("rwoption7", 43)
except Exception as e:
assert isinstance(e, ValueError)
else:
message = "should raise if value is not in specified range"
assert False, message
# Specified module does not exist => return None.
assert self.get_module_option_ex("foo", "bar") is None
# Specified key does not exist => return None.
assert self.get_module_option_ex("dashboard", "bar") is None
self.set_module_option_ex("telemetry", "contact", "*****@*****.**")
assert self.get_module_option_ex("telemetry", "contact") == "*****@*****.**"
# No option default value, so use the specified one.
assert self.get_module_option_ex("dashboard", "password") is None
assert self.get_module_option_ex("dashboard", "password", "foobar") == "foobar"
# Option type is not defined => return as string.
self.set_module_option_ex("selftest", "rwoption1", 1234)
value = self.get_module_option_ex("selftest", "rwoption1")
assert isinstance(value, str)
assert value == "1234"
# Option type is defined => return as integer.
self.set_module_option_ex("telemetry", "interval", 60)
value = self.get_module_option_ex("telemetry", "interval")
assert isinstance(value, int)
assert value == 60
# Option type is defined => return as bool.
self.set_module_option_ex("telemetry", "leaderboard", True)
value = self.get_module_option_ex("telemetry", "leaderboard")
assert isinstance(value, bool)
assert value is True
def _self_test_store(self) -> None:
existing_keys = set(self.get_store_prefix("test").keys())
self.set_store("testkey", "testvalue")
assert self.get_store("testkey") == "testvalue"
assert (set(self.get_store_prefix("test").keys())
== {"testkey"} | existing_keys)
def _self_test_perf_counters(self) -> None:
self.get_perf_schema("osd", "0")
self.get_counter("osd", "0", "osd.op")
# get_counter
# get_all_perf_coutners
def _self_test_misc(self) -> None:
self.set_uri("http://this.is.a.test.com")
self.set_health_checks({})
def _self_test_osdmap(self) -> None:
osdmap = self.get_osdmap()
osdmap.get_epoch()
osdmap.get_crush_version()
osdmap.dump()
inc = osdmap.new_incremental()
osdmap.apply_incremental(inc)
inc.get_epoch()
inc.dump()
crush = osdmap.get_crush()
crush.dump()
crush.get_item_name(-1)
crush.get_item_weight(-1)
crush.find_takes()
crush.get_take_weight_osd_map(-1)
# osdmap.get_pools_by_take()
# osdmap.calc_pg_upmaps()
# osdmap.map_pools_pgs_up()
# inc.set_osd_reweights
# inc.set_crush_compat_weight_set_weights
self.log.info("Finished self-test procedure.")
def _test_remote_calls(self) -> None:
# Test making valid call
self.remote("influx", "self_test")
# Test calling module that exists but isn't enabled
# (arbitrarily pick a non-always-on module to use)
disabled_module = "telegraf"
mgr_map = self.get("mgr_map")
assert disabled_module not in mgr_map['modules']
# (This works until the Z release in about 2027)
latest_release = sorted(mgr_map['always_on_modules'].keys())[-1]
assert disabled_module not in mgr_map['always_on_modules'][latest_release]
try:
self.remote(disabled_module, "handle_command", {"prefix": "influx self-test"})
except ImportError:
pass
else:
raise RuntimeError("ImportError not raised for disabled module")
# Test calling module that doesn't exist
try:
self.remote("idontexist", "self_test")
except ImportError:
pass
else:
raise RuntimeError("ImportError not raised for nonexistent module")
# Test calling method that doesn't exist
try:
self.remote("influx", "idontexist")
except NameError:
pass
else:
raise RuntimeError("KeyError not raised")
def remote_from_orchestrator_cli_self_test(self, what: str) -> Any:
import orchestrator
if what == 'OrchestratorError':
return orchestrator.OrchResult(result=None, exception=orchestrator.OrchestratorError('hello, world'))
elif what == "ZeroDivisionError":
return orchestrator.OrchResult(result=None, exception=ZeroDivisionError('hello, world'))
assert False, repr(what)
def shutdown(self) -> None:
self._workload = Workload.SHUTDOWN
self._event.set()
def _command_spam(self) -> None:
self.log.info("Starting command_spam workload...")
while not self._event.is_set():
osdmap = self.get_osdmap()
dump = osdmap.dump()
count = len(dump['osds'])
i = int(random.random() * count)
w = random.random()
result = CommandResult('')
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'osd reweight',
'id': i,
'weight': w}), '')
_ = osdmap.get_crush().dump()
r, outb, outs = result.wait()
self._event.clear()
self.log.info("Ended command_spam workload...")
@CLICommand('mgr self-test eval')
def eval(self,
s: Optional[str] = None,
inbuf: Optional[str] = None) -> HandleCommandResult:
'''
eval given source
'''
source = s or inbuf
if source is None:
return HandleCommandResult(-1, '', 'source is not specified')
err = StringIO()
out = StringIO()
with redirect_stderr(err), redirect_stdout(out):
needs_more = self._repl.runsource(source)
if needs_more:
retval = 2
stdout = ''
stderr = ''
else:
retval = 0
stdout = out.getvalue()
stderr = err.getvalue()
return HandleCommandResult(retval, stdout, stderr)
def serve(self) -> None:
while True:
if self._workload == Workload.COMMAND_SPAM:
self._command_spam()
elif self._workload == Workload.SHUTDOWN:
self.log.info("Shutting down...")
break
elif self._workload == Workload.THROW_EXCEPTION:
raise RuntimeError("Synthetic exception in serve")
else:
self.log.info("Waiting for workload request...")
self._event.wait()
self._event.clear()
class Module(MgrModule):
MODULE_OPTIONS = [
Option(name='warn_recent_interval',
type='secs',
default=60 * 60 * 24 * 14,
desc='time interval in which to warn about recent crashes',
runtime=True),
Option(name='retain_interval',
type='secs',
default=60 * 60 * 24 * 365,
desc='how long to retain crashes before pruning them',
runtime=True),
]
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
self.crashes: Optional[Dict[str, CrashT]] = None
self.crashes_lock = Lock()
self.run = True
self.event = Event()
if TYPE_CHECKING:
self.warn_recent_interval = 0.0
self.retain_interval = 0.0
def shutdown(self) -> None:
self.run = False
self.event.set()
def serve(self) -> None:
self.config_notify()
while self.run:
with self.crashes_lock:
self._refresh_health_checks()
self._prune(self.retain_interval)
wait = min(MAX_WAIT, max(self.warn_recent_interval / 100,
MIN_WAIT))
self.event.wait(wait)
self.event.clear()
def config_notify(self) -> None:
for opt in self.MODULE_OPTIONS:
setattr(self, opt['name'], self.get_module_option(opt['name']))
self.log.debug(' mgr option %s = %s', opt['name'],
getattr(self, opt['name']))
def _load_crashes(self) -> None:
raw = self.get_store_prefix('crash/')
self.crashes = {k[6:]: json.loads(m) for (k, m) in raw.items()}
def _refresh_health_checks(self) -> None:
if not self.crashes:
self._load_crashes()
assert self.crashes is not None
cutoff = datetime.datetime.utcnow() - datetime.timedelta(
seconds=self.warn_recent_interval)
recent = {
crashid: crash
for crashid, crash in self.crashes.items()
if (self.time_from_string(cast(str, crash['timestamp'])) > cutoff
and 'archived' not in crash)
}
num = len(recent)
health_checks: Dict[str, Dict[str, Union[int, str, List[str]]]] = {}
if recent:
detail = [
'%s crashed on host %s at %s' %
(crash.get('entity_name', 'unidentified daemon'),
crash.get('utsname_hostname', '(unknown)'),
crash.get('timestamp', 'unknown time'))
for crash in recent.values()
]
if num > 30:
detail = detail[0:30]
detail.append('and %d more' % (num - 30))
self.log.debug('detail %s' % detail)
health_checks['RECENT_CRASH'] = {
'severity': 'warning',
'summary': '%d daemons have recently crashed' % (num),
'count': num,
'detail': detail,
}
self.set_health_checks(health_checks)
def time_from_string(self, timestr: str) -> datetime.datetime:
# drop the 'Z' timezone indication, it's always UTC
timestr = timestr.rstrip('Z')
try:
return datetime.datetime.strptime(timestr, DATEFMT)
except ValueError:
return datetime.datetime.strptime(timestr, OLD_DATEFMT)
def validate_crash_metadata(
self, inbuf: str) -> Dict[str, Union[str, List[str]]]:
# raise any exceptions to caller
metadata = json.loads(inbuf)
for f in ['crash_id', 'timestamp']:
if f not in metadata:
raise AttributeError("missing '%s' field" % f)
_ = self.time_from_string(metadata['timestamp'])
return metadata
def timestamp_filter(
self, f: Callable[[datetime.datetime],
bool]) -> Iterable[Tuple[str, CrashT]]:
"""
Filter crash reports by timestamp.
:param f: f(time) return true to keep crash report
:returns: crash reports for which f(time) returns true
"""
def inner(pair: Tuple[str, CrashT]) -> bool:
_, crash = pair
time = self.time_from_string(cast(str, crash["timestamp"]))
return f(time)
assert self.crashes is not None
return filter(inner, self.crashes.items())
# stack signature helpers
def sanitize_backtrace(self, bt: List[str]) -> List[str]:
ret = list()
for func_record in bt:
# split into two fields on last space, take the first one,
# strip off leading ( and trailing )
func_plus_offset = func_record.rsplit(' ', 1)[0][1:-1]
ret.append(func_plus_offset.split('+')[0])
return ret
ASSERT_MATCHEXPR = re.compile(r'(?s)(.*) thread .* time .*(: .*)\n')
def sanitize_assert_msg(self, msg: str) -> str:
# (?s) allows matching newline. get everything up to "thread" and
# then after-and-including the last colon-space. This skips the
# thread id, timestamp, and file:lineno, because file is already in
# the beginning, and lineno may vary.
matched = self.ASSERT_MATCHEXPR.match(msg)
assert matched
return ''.join(matched.groups())
def calc_sig(self, bt: List[str], assert_msg: Optional[str]) -> str:
sig = hashlib.sha256()
for func in self.sanitize_backtrace(bt):
sig.update(func.encode())
if assert_msg:
sig.update(self.sanitize_assert_msg(assert_msg).encode())
return ''.join('%02x' % c for c in sig.digest())
# command handlers
@CLIReadCommand('crash info')
@with_crashes
def do_info(self, id: str) -> Tuple[int, str, str]:
"""
show crash dump metadata
"""
crashid = id
assert self.crashes is not None
crash = self.crashes.get(crashid)
if not crash:
return errno.EINVAL, '', 'crash info: %s not found' % crashid
val = json.dumps(crash, indent=4, sort_keys=True)
return 0, val, ''
@CLICommand('crash post')
def do_post(self, inbuf: str) -> Tuple[int, str, str]:
"""
Add a crash dump (use -i <jsonfile>)
"""
try:
metadata = self.validate_crash_metadata(inbuf)
except Exception as e:
return errno.EINVAL, '', 'malformed crash metadata: %s' % e
if 'backtrace' in metadata:
backtrace = cast(List[str], metadata.get('backtrace'))
assert_msg = cast(Optional[str], metadata.get('assert_msg'))
metadata['stack_sig'] = self.calc_sig(backtrace, assert_msg)
crashid = cast(str, metadata['crash_id'])
assert self.crashes is not None
if crashid not in self.crashes:
self.crashes[crashid] = metadata
key = 'crash/%s' % crashid
self.set_store(key, json.dumps(metadata))
self._refresh_health_checks()
return 0, '', ''
def ls(self) -> Tuple[int, str, str]:
if not self.crashes:
self._load_crashes()
return self.do_ls_all('')
def _do_ls(self, t: Iterable[CrashT],
format: Optional[str]) -> Tuple[int, str, str]:
r = sorted(t, key=lambda i: i['crash_id'])
if format in ('json', 'json-pretty'):
return 0, json.dumps(r, indent=4, sort_keys=True), ''
else:
table = PrettyTable(['ID', 'ENTITY', 'NEW'], border=False)
table.left_padding_width = 0
table.right_padding_width = 2
table.align['ID'] = 'l'
table.align['ENTITY'] = 'l'
for c in r:
table.add_row([
c.get('crash_id'),
c.get('entity_name', 'unknown'),
'' if 'archived' in c else '*'
])
return 0, table.get_string(), ''
@CLIReadCommand('crash ls')
@with_crashes
def do_ls_all(self, format: Optional[str] = None) -> Tuple[int, str, str]:
"""
Show new and archived crash dumps
"""
assert self.crashes is not None
return self._do_ls(self.crashes.values(), format)
@CLIReadCommand('crash ls-new')
@with_crashes
def do_ls_new(self, format: Optional[str] = None) -> Tuple[int, str, str]:
"""
Show new crash dumps
"""
assert self.crashes is not None
t = [
crash for crashid, crash in self.crashes.items()
if 'archived' not in crash
]
return self._do_ls(t, format)
@CLICommand('crash rm')
@with_crashes
def do_rm(self, id: str) -> Tuple[int, str, str]:
"""
Remove a saved crash <id>
"""
crashid = id
assert self.crashes is not None
if crashid in self.crashes:
del self.crashes[crashid]
key = 'crash/%s' % crashid
self.set_store(key, None) # removes key
self._refresh_health_checks()
return 0, '', ''
@CLICommand('crash prune')
@with_crashes
def do_prune(self, keep: int) -> Tuple[int, str, str]:
"""
Remove crashes older than <keep> days
"""
self._prune(keep * datetime.timedelta(days=1).total_seconds())
return 0, '', ''
def _prune(self, seconds: float) -> None:
now = datetime.datetime.utcnow()
cutoff = now - datetime.timedelta(seconds=seconds)
removed_any = False
# make a copy of the list, since we'll modify self.crashes below
to_prune = list(self.timestamp_filter(lambda ts: ts <= cutoff))
assert self.crashes is not None
for crashid, crash in to_prune:
del self.crashes[crashid]
key = 'crash/%s' % crashid
self.set_store(key, None)
removed_any = True
if removed_any:
self._refresh_health_checks()
@CLIWriteCommand('crash archive')
@with_crashes
def do_archive(self, id: str) -> Tuple[int, str, str]:
"""
Acknowledge a crash and silence health warning(s)
"""
crashid = id
assert self.crashes is not None
crash = self.crashes.get(crashid)
if not crash:
return errno.EINVAL, '', 'crash info: %s not found' % crashid
if not crash.get('archived'):
crash['archived'] = str(datetime.datetime.utcnow())
self.crashes[crashid] = crash
key = 'crash/%s' % crashid
self.set_store(key, json.dumps(crash))
self._refresh_health_checks()
return 0, '', ''
@CLIWriteCommand('crash archive-all')
@with_crashes
def do_archive_all(self) -> Tuple[int, str, str]:
"""
Acknowledge all new crashes and silence health warning(s)
"""
assert self.crashes is not None
for crashid, crash in self.crashes.items():
if not crash.get('archived'):
crash['archived'] = str(datetime.datetime.utcnow())
self.crashes[crashid] = crash
key = 'crash/%s' % crashid
self.set_store(key, json.dumps(crash))
self._refresh_health_checks()
return 0, '', ''
@CLIReadCommand('crash stat')
@with_crashes
def do_stat(self) -> Tuple[int, str, str]:
"""
Summarize recorded crashes
"""
# age in days for reporting, ordered smallest first
AGE_IN_DAYS = [1, 3, 7]
retlines = list()
BinnedStatsT = Dict[str, Union[int, datetime.datetime, List[str]]]
def binstr(bindict: BinnedStatsT) -> str:
binlines = list()
id_list = cast(List[str], bindict['idlist'])
count = len(id_list)
if count:
binlines.append('%d older than %s days old:' %
(count, bindict['age']))
for crashid in id_list:
binlines.append(crashid)
return '\n'.join(binlines)
total = 0
now = datetime.datetime.utcnow()
bins: List[BinnedStatsT] = []
for age in AGE_IN_DAYS:
agelimit = now - datetime.timedelta(days=age)
bins.append({'age': age, 'agelimit': agelimit, 'idlist': list()})
assert self.crashes is not None
for crashid, crash in self.crashes.items():
total += 1
stamp = self.time_from_string(cast(str, crash['timestamp']))
for bindict in bins:
if stamp <= cast(datetime.datetime, bindict['agelimit']):
cast(List[str], bindict['idlist']).append(crashid)
# don't count this one again
continue
retlines.append('%d crashes recorded' % total)
for bindict in bins:
retlines.append(binstr(bindict))
return 0, '\n'.join(retlines), ''
@CLIReadCommand('crash json_report')
@with_crashes
def do_json_report(self, hours: int) -> Tuple[int, str, str]:
"""
Crashes in the last <hours> hours
"""
# Return a machine readable summary of recent crashes.
report: DefaultDict[str, int] = defaultdict(lambda: 0)
assert self.crashes is not None
for crashid, crash in self.crashes.items():
pname = cast(str, crash.get("process_name", "unknown"))
if not pname:
pname = "unknown"
report[pname] += 1
return 0, '', json.dumps(report, sort_keys=True)
def self_test(self) -> None:
# test time conversion
timestr = '2018-06-22T20:35:38.058818Z'
old_timestr = '2018-06-22 20:35:38.058818Z'
dt = self.time_from_string(timestr)
if dt != datetime.datetime(2018, 6, 22, 20, 35, 38, 58818):
raise RuntimeError('time_from_string() failed')
dt = self.time_from_string(old_timestr)
if dt != datetime.datetime(2018, 6, 22, 20, 35, 38, 58818):
raise RuntimeError('time_from_string() (old) failed')
class Alerts(MgrModule):
MODULE_OPTIONS = [
Option(name='interval',
type='secs',
default=60,
desc='How frequently to reexamine health status',
runtime=True),
# smtp
Option(name='smtp_host', default='', desc='SMTP server', runtime=True),
Option(name='smtp_destination',
default='',
desc='Email address to send alerts to',
runtime=True),
Option(name='smtp_port',
type='int',
default=465,
desc='SMTP port',
runtime=True),
Option(name='smtp_ssl',
type='bool',
default=True,
desc='Use SSL to connect to SMTP server',
runtime=True),
Option(name='smtp_user',
default='',
desc='User to authenticate as',
runtime=True),
Option(name='smtp_password',
default='',
desc='Password to authenticate with',
runtime=True),
Option(name='smtp_sender',
default='',
desc='SMTP envelope sender',
runtime=True),
Option(name='smtp_from_name',
default='Ceph',
desc='Email From: name',
runtime=True)
]
# These are "native" Ceph options that this module cares about.
NATIVE_OPTIONS: List[str] = []
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Alerts, self).__init__(*args, **kwargs)
# set up some members to enable the serve() method and shutdown()
self.run = True
self.event = Event()
# ensure config options members are initialized; see config_notify()
self.config_notify()
self.log.info("Init")
if TYPE_CHECKING:
self.interval = 60
self.smtp_host = ''
self.smtp_destination = ''
self.smtp_port = 0
self.smtp_ssl = True
self.smtp_user = ''
self.smtp_password = ''
self.smtp_sender = ''
self.smtp_from_name = ''
def config_notify(self) -> None:
"""
This method is called whenever one of our config options is changed.
"""
# This is some boilerplate that stores MODULE_OPTIONS in a class
# member, so that, for instance, the 'emphatic' option is always
# available as 'self.emphatic'.
for opt in self.MODULE_OPTIONS:
setattr(self, opt['name'], self.get_module_option(opt['name']))
self.log.debug(' mgr option %s = %s', opt['name'],
getattr(self, opt['name']))
# Do the same for the native options.
for opt in self.NATIVE_OPTIONS:
setattr(self, opt, self.get_ceph_option(opt))
self.log.debug(' native option %s = %s', opt, getattr(self, opt))
@CLIReadCommand('alerts send')
def send(self) -> HandleCommandResult:
"""
(re)send alerts immediately
"""
status = json.loads(self.get('health')['json'])
self._send_alert(status, {})
return HandleCommandResult()
def _diff(self, last: Dict[str, Any], new: Dict[str,
Any]) -> Dict[str, Any]:
d: Dict[str, Any] = {}
for code, alert in new.get('checks', {}).items():
self.log.debug('new code %s alert %s' % (code, alert))
if code not in last.get('checks', {}):
if 'new' not in d:
d['new'] = {}
d['new'][code] = alert
elif (alert['summary'].get('count', 0) >
last['checks'][code]['summary'].get('count', 0)):
if 'updated' not in d:
d['updated'] = {}
d['updated'][code] = alert
for code, alert in last.get('checks', {}).items():
self.log.debug('old code %s alert %s' % (code, alert))
if code not in new.get('checks', {}):
if 'cleared' not in d:
d['cleared'] = {}
d['cleared'][code] = alert
return d
def _send_alert(self, status: Dict[str, Any], diff: Dict[str,
Any]) -> None:
checks = {}
if self.smtp_host:
r = self._send_alert_smtp(status, diff)
if r:
for code, alert in r.items():
checks[code] = alert
else:
self.log.warning(
'Alert is not sent because smtp_host is not configured')
self.set_health_checks(checks)
def serve(self) -> None:
"""
This method is called by the mgr when the module starts and can be
used for any background activity.
"""
self.log.info("Starting")
last_status: Dict[str, Any] = {}
while self.run:
# Do some useful background work here.
new_status = json.loads(self.get('health')['json'])
if new_status != last_status:
self.log.debug('last_status %s' % last_status)
self.log.debug('new_status %s' % new_status)
diff = self._diff(last_status, new_status)
self.log.debug('diff %s' % diff)
if diff:
self._send_alert(new_status, diff)
last_status = new_status
self.log.debug('Sleeping for %s seconds', self.interval)
self.event.wait(self.interval or 60)
self.event.clear()
def shutdown(self) -> None:
"""
This method is called by the mgr when the module needs to shut
down (i.e., when the serve() function needs to exit).
"""
self.log.info('Stopping')
self.run = False
self.event.set()
# SMTP
def _smtp_format_alert(self, code: str, alert: Dict[str, Any]) -> str:
r = '[{sev}] {code}: {summary}\n'.format(
code=code,
sev=alert['severity'].split('_')[1],
summary=alert['summary']['message'])
for detail in alert['detail']:
r += ' {message}\n'.format(message=detail['message'])
return r
def _send_alert_smtp(self, status: Dict[str, Any],
diff: Dict[str, Any]) -> Optional[Dict[str, Any]]:
# message
self.log.debug('_send_alert_smtp')
message = ('From: {from_name} <{sender}>\n'
'Subject: {status}\n'
'To: {target}\n'
'\n'
'{status}\n'.format(sender=self.smtp_sender,
from_name=self.smtp_from_name,
status=status['status'],
target=self.smtp_destination))
if 'new' in diff:
message += ('\n--- New ---\n')
for code, alert in diff['new'].items():
message += self._smtp_format_alert(code, alert)
if 'updated' in diff:
message += ('\n--- Updated ---\n')
for code, alert in diff['updated'].items():
message += self._smtp_format_alert(code, alert)
if 'cleared' in diff:
message += ('\n--- Cleared ---\n')
for code, alert in diff['cleared'].items():
message += self._smtp_format_alert(code, alert)
message += ('\n\n=== Full health status ===\n')
for code, alert in status['checks'].items():
message += self._smtp_format_alert(code, alert)
self.log.debug('message: %s' % message)
# send
try:
if self.smtp_ssl:
server: Union[smtplib.SMTP_SSL, smtplib.SMTP] = \
smtplib.SMTP_SSL(self.smtp_host, self.smtp_port)
else:
server = smtplib.SMTP(self.smtp_host, self.smtp_port)
if self.smtp_password:
server.login(self.smtp_user, self.smtp_password)
server.sendmail(self.smtp_sender, self.smtp_destination, message)
server.quit()
except Exception as e:
return {
'ALERTS_SMTP_ERROR': {
'severity': 'warning',
'summary': 'unable to send alert email',
'count': 1,
'detail': [str(e)]
}
}
self.log.debug('Sent email to %s' % self.smtp_destination)
return None
class Module(MgrModule):
MODULE_OPTIONS = [
Option(name='address', default='unixgram:///tmp/telegraf.sock'),
Option(name='interval', type='secs', default=15)
]
ceph_health_mapping = {'HEALTH_OK': 0, 'HEALTH_WARN': 1, 'HEALTH_ERR': 2}
@property
def config_keys(self) -> Dict[str, OptionValue]:
return dict(
(o['name'], o.get('default', None)) for o in self.MODULE_OPTIONS)
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
self.event = Event()
self.run = True
self.fsid: Optional[str] = None
self.config: Dict[str, OptionValue] = dict()
def get_fsid(self) -> str:
if not self.fsid:
self.fsid = self.get('mon_map')['fsid']
assert self.fsid is not None
return self.fsid
def get_pool_stats(self) -> Iterable[Dict[str, Any]]:
df = self.get('df')
df_types = [
'bytes_used', 'kb_used', 'dirty', 'rd', 'rd_bytes', 'stored_raw',
'wr', 'wr_bytes', 'objects', 'max_avail', 'quota_objects',
'quota_bytes'
]
for df_type in df_types:
for pool in df['pools']:
yield {
'measurement': 'ceph_pool_stats',
'tags': {
'pool_name': pool['name'],
'pool_id': pool['id'],
'type_instance': df_type,
'fsid': self.get_fsid()
},
'value': pool['stats'][df_type],
}
def get_daemon_stats(self) -> Iterable[Dict[str, Any]]:
for daemon, counters in self.get_all_perf_counters().items():
svc_type, svc_id = daemon.split('.', 1)
metadata = self.get_metadata(svc_type, svc_id)
if not metadata:
continue
for path, counter_info in counters.items():
if counter_info['type'] & self.PERFCOUNTER_HISTOGRAM:
continue
yield {
'measurement': 'ceph_daemon_stats',
'tags': {
'ceph_daemon': daemon,
'type_instance': path,
'host': metadata['hostname'],
'fsid': self.get_fsid()
},
'value': counter_info['value']
}
def get_pg_stats(self) -> Dict[str, int]:
stats = dict()
pg_status = self.get('pg_status')
for key in [
'bytes_total', 'data_bytes', 'bytes_used', 'bytes_avail',
'num_pgs', 'num_objects', 'num_pools'
]:
stats[key] = pg_status[key]
for state in PG_STATES:
stats['num_pgs_{0}'.format(state)] = 0
stats['num_pgs'] = pg_status['num_pgs']
for state in pg_status['pgs_by_state']:
states = state['state_name'].split('+')
for s in PG_STATES:
key = 'num_pgs_{0}'.format(s)
if s in states:
stats[key] += state['count']
return stats
def get_cluster_stats(self) -> Iterable[Dict[str, Any]]:
stats = dict()
health = json.loads(self.get('health')['json'])
stats['health'] = self.ceph_health_mapping.get(health['status'])
mon_status = json.loads(self.get('mon_status')['json'])
stats['num_mon'] = len(mon_status['monmap']['mons'])
stats['mon_election_epoch'] = mon_status['election_epoch']
stats['mon_outside_quorum'] = len(mon_status['outside_quorum'])
stats['mon_quorum'] = len(mon_status['quorum'])
osd_map = self.get('osd_map')
stats['num_osd'] = len(osd_map['osds'])
stats['num_pg_temp'] = len(osd_map['pg_temp'])
stats['osd_epoch'] = osd_map['epoch']
mgr_map = self.get('mgr_map')
stats['mgr_available'] = int(mgr_map['available'])
stats['num_mgr_standby'] = len(mgr_map['standbys'])
stats['mgr_epoch'] = mgr_map['epoch']
num_up = 0
num_in = 0
for osd in osd_map['osds']:
if osd['up'] == 1:
num_up += 1
if osd['in'] == 1:
num_in += 1
stats['num_osd_up'] = num_up
stats['num_osd_in'] = num_in
fs_map = self.get('fs_map')
stats['num_mds_standby'] = len(fs_map['standbys'])
stats['num_fs'] = len(fs_map['filesystems'])
stats['mds_epoch'] = fs_map['epoch']
num_mds_up = 0
for fs in fs_map['filesystems']:
num_mds_up += len(fs['mdsmap']['up'])
stats['num_mds_up'] = num_mds_up
stats['num_mds'] = num_mds_up + cast(int, stats['num_mds_standby'])
stats.update(self.get_pg_stats())
for key, value in stats.items():
assert value is not None
yield {
'measurement': 'ceph_cluster_stats',
'tags': {
'type_instance': key,
'fsid': self.get_fsid()
},
'value': int(value)
}
def set_config_option(self, option: str, value: str) -> None:
if option not in self.config_keys.keys():
raise RuntimeError('{0} is a unknown configuration '
'option'.format(option))
if option == 'interval':
try:
interval = int(value)
except (ValueError, TypeError):
raise RuntimeError('invalid {0} configured. Please specify '
'a valid integer'.format(option))
if interval < 5:
raise RuntimeError(
'interval should be set to at least 5 seconds')
self.config[option] = interval
else:
self.config[option] = value
def init_module_config(self) -> None:
self.config['address'] = \
self.get_module_option("address", default=self.config_keys['address'])
interval = self.get_module_option("interval",
default=self.config_keys['interval'])
assert interval
self.config['interval'] = int(interval)
def now(self) -> int:
return int(round(time.time() * 1000000000))
def gather_measurements(self) -> Iterable[Dict[str, Any]]:
return itertools.chain(self.get_pool_stats(), self.get_daemon_stats(),
self.get_cluster_stats())
def send_to_telegraf(self) -> None:
url = urlparse(cast(str, self.config['address']))
sock = BaseSocket(url)
self.log.debug('Sending data to Telegraf at %s', sock.address)
now = self.now()
try:
with sock as s:
for measurement in self.gather_measurements():
self.log.debug(measurement)
line = Line(measurement['measurement'],
measurement['value'], measurement['tags'], now)
self.log.debug(line.to_line_protocol())
s.send(line.to_line_protocol())
except (socket.error, RuntimeError, IOError, OSError):
self.log.exception('Failed to send statistics to Telegraf:')
except FileNotFoundError:
self.log.exception('Failed to open Telegraf at: %s', url.geturl())
def shutdown(self) -> None:
self.log.info('Stopping Telegraf module')
self.run = False
self.event.set()
@CLIReadCommand('telegraf config-show')
def config_show(self) -> Tuple[int, str, str]:
"""
Show current configuration
"""
return 0, json.dumps(self.config), ''
@CLICommand('telegraf config-set')
def config_set(self, key: str, value: str) -> Tuple[int, str, str]:
"""
Set a configuration value
"""
if not value:
return -errno.EINVAL, '', 'Value should not be empty or None'
self.log.debug('Setting configuration option %s to %s', key, value)
self.set_config_option(key, value)
self.set_module_option(key, value)
return 0, 'Configuration option {0} updated'.format(key), ''
@CLICommand('telegraf send')
def send(self) -> Tuple[int, str, str]:
"""
Force sending data to Telegraf
"""
self.send_to_telegraf()
return 0, 'Sending data to Telegraf', ''
def self_test(self) -> None:
measurements = list(self.gather_measurements())
if len(measurements) == 0:
raise RuntimeError('No measurements found')
def serve(self) -> None:
self.log.info('Starting Telegraf module')
self.init_module_config()
self.run = True
self.log.debug('Waiting 10 seconds before starting')
self.event.wait(10)
while self.run:
start = self.now()
self.send_to_telegraf()
runtime = (self.now() - start) / 1000000
self.log.debug('Sending data to Telegraf took %d ms', runtime)
self.log.debug("Sleeping for %d seconds", self.config['interval'])
self.event.wait(cast(int, self.config['interval']))
class Module(MgrModule):
# latest (if db does not exist)
SCHEMA = """
CREATE TABLE Device (
devid TEXT PRIMARY KEY
) WITHOUT ROWID;
CREATE TABLE DeviceHealthMetrics (
time DATETIME DEFAULT (strftime('%s', 'now')),
devid TEXT NOT NULL REFERENCES Device (devid),
raw_smart TEXT NOT NULL,
PRIMARY KEY (time, devid)
);
"""
SCHEMA_VERSIONED = [
# v1
"""
CREATE TABLE Device (
devid TEXT PRIMARY KEY
) WITHOUT ROWID;
CREATE TABLE DeviceHealthMetrics (
time DATETIME DEFAULT (strftime('%s', 'now')),
devid TEXT NOT NULL REFERENCES Device (devid),
raw_smart TEXT NOT NULL,
PRIMARY KEY (time, devid)
);
"""
]
MODULE_OPTIONS = [
Option(
name='enable_monitoring',
default=True,
type='bool',
desc='monitor device health metrics',
runtime=True,
),
Option(
name='scrape_frequency',
default=86400,
type='secs',
desc='how frequently to scrape device health metrics',
runtime=True,
),
Option(
name='pool_name',
default='device_health_metrics',
type='str',
desc='name of pool in which to store device health metrics',
runtime=True,
),
Option(
name='retention_period',
default=(86400 * 180),
type='secs',
desc='how long to retain device health metrics',
runtime=True,
),
Option(
name='mark_out_threshold',
default=(86400 * 14 * 2),
type='secs',
desc='automatically mark OSD if it may fail before this long',
runtime=True,
),
Option(
name='warn_threshold',
default=(86400 * 14 * 6),
type='secs',
desc='raise health warning if OSD may fail before this long',
runtime=True,
),
Option(
name='self_heal',
default=True,
type='bool',
desc='preemptively heal cluster around devices that may fail',
runtime=True,
),
Option(
name='sleep_interval',
default=600,
type='secs',
desc='how frequently to wake up and check device health',
runtime=True,
),
]
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
# populate options (just until serve() runs)
for opt in self.MODULE_OPTIONS:
setattr(self, opt['name'], opt['default'])
# other
self.run = True
self.event = Event()
# for mypy which does not run the code
if TYPE_CHECKING:
self.enable_monitoring = True
self.scrape_frequency = 0.0
self.pool_name = ''
self.device_health_metrics = ''
self.retention_period = 0.0
self.mark_out_threshold = 0.0
self.warn_threshold = 0.0
self.self_heal = True
self.sleep_interval = 0.0
def is_valid_daemon_name(self, who: str) -> bool:
parts = who.split('.')
if len(parts) != 2:
return False
return parts[0] in ('osd', 'mon')
@CLICommand('device query-daemon-health-metrics', perm='r')
def do_query_daemon_health_metrics(self, who: str) -> Tuple[int, str, str]:
'''
Get device health metrics for a given daemon
'''
if not self.is_valid_daemon_name(who):
return -errno.EINVAL, '', 'not a valid mon or osd daemon name'
(daemon_type, daemon_id) = who.split('.')
result = CommandResult('')
self.send_command(result, daemon_type, daemon_id,
json.dumps({
'prefix': 'smart',
'format': 'json',
}), '')
return result.wait()
@CLIRequiresDB
@CLICommand('device scrape-daemon-health-metrics', perm='r')
def do_scrape_daemon_health_metrics(self,
who: str) -> Tuple[int, str, str]:
'''
Scrape and store device health metrics for a given daemon
'''
if not self.is_valid_daemon_name(who):
return -errno.EINVAL, '', 'not a valid mon or osd daemon name'
(daemon_type, daemon_id) = who.split('.')
return self.scrape_daemon(daemon_type, daemon_id)
@CLIRequiresDB
@CLICommand('device scrape-health-metrics', perm='r')
def do_scrape_health_metrics(self,
devid: Optional[str] = None
) -> Tuple[int, str, str]:
'''
Scrape and store device health metrics
'''
if devid is None:
return self.scrape_all()
else:
return self.scrape_device(devid)
@CLIRequiresDB
@CLICommand('device get-health-metrics', perm='r')
def do_get_health_metrics(
self,
devid: str,
sample: Optional[str] = None) -> Tuple[int, str, str]:
'''
Show stored device metrics for the device
'''
return self.show_device_metrics(devid, sample)
@CLIRequiresDB
@CLICommand('device check-health', perm='rw')
def do_check_health(self) -> Tuple[int, str, str]:
'''
Check life expectancy of devices
'''
return self.check_health()
@CLICommand('device monitoring on', perm='rw')
def do_monitoring_on(self) -> Tuple[int, str, str]:
'''
Enable device health monitoring
'''
self.set_module_option('enable_monitoring', True)
self.event.set()
return 0, '', ''
@CLICommand('device monitoring off', perm='rw')
def do_monitoring_off(self) -> Tuple[int, str, str]:
'''
Disable device health monitoring
'''
self.set_module_option('enable_monitoring', False)
self.set_health_checks({}) # avoid stuck health alerts
return 0, '', ''
@CLIRequiresDB
@CLICommand('device predict-life-expectancy', perm='r')
def do_predict_life_expectancy(self, devid: str) -> Tuple[int, str, str]:
'''
Predict life expectancy with local predictor
'''
return self.predict_lift_expectancy(devid)
def self_test(self) -> None:
assert self.db_ready()
self.config_notify()
osdmap = self.get('osd_map')
osd_id = osdmap['osds'][0]['osd']
osdmeta = self.get('osd_metadata')
devs = osdmeta.get(str(osd_id), {}).get('device_ids')
if devs:
devid = devs.split()[0].split('=')[1]
self.log.debug(f"getting devid {devid}")
(r, before, err) = self.show_device_metrics(devid, None)
assert r == 0
self.log.debug(f"before: {before}")
(r, out, err) = self.scrape_device(devid)
assert r == 0
(r, after, err) = self.show_device_metrics(devid, None)
assert r == 0
self.log.debug(f"after: {after}")
assert before != after
def config_notify(self) -> None:
for opt in self.MODULE_OPTIONS:
setattr(self, opt['name'], self.get_module_option(opt['name']))
self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name']))
def _legacy_put_device_metrics(self, t: str, devid: str,
data: str) -> None:
SQL = """
INSERT OR IGNORE INTO DeviceHealthMetrics (time, devid, raw_smart)
VALUES (?, ?, ?);
"""
self._create_device(devid)
epoch = self._t2epoch(t)
json.loads(data) # valid?
self.db.execute(SQL, (epoch, devid, data))
devre = r"[a-zA-Z0-9-]+[_-][a-zA-Z0-9-]+[_-][a-zA-Z0-9-]+"
def _load_legacy_object(self, ioctx: rados.Ioctx, oid: str) -> bool:
MAX_OMAP = 10000
self.log.debug(f"loading object {oid}")
if re.search(self.devre, oid) is None:
return False
with rados.ReadOpCtx() as op:
it, rc = ioctx.get_omap_vals(op, None, None, MAX_OMAP)
if rc == 0:
ioctx.operate_read_op(op, oid)
count = 0
for t, raw_smart in it:
self.log.debug(f"putting {oid} {t}")
self._legacy_put_device_metrics(t, oid, raw_smart)
count += 1
assert count < MAX_OMAP
self.log.debug(f"removing object {oid}")
ioctx.remove_object(oid)
return True
def check_legacy_pool(self) -> bool:
try:
# 'device_health_metrics' is automatically renamed '.mgr' in
# create_mgr_pool
ioctx = self.rados.open_ioctx(self.MGR_POOL_NAME)
except rados.ObjectNotFound:
return True
if not ioctx:
return True
done = False
with ioctx, self._db_lock, self.db:
count = 0
for obj in ioctx.list_objects():
try:
if self._load_legacy_object(ioctx, obj.key):
count += 1
except json.decoder.JSONDecodeError:
pass
if count >= 10:
break
done = count < 10
self.log.debug(f"finished reading legacy pool, complete = {done}")
return done
def serve(self) -> None:
self.log.info("Starting")
self.config_notify()
last_scrape = None
finished_loading_legacy = False
while self.run:
if self.db_ready() and self.enable_monitoring:
self.log.debug('Running')
if not finished_loading_legacy:
finished_loading_legacy = self.check_legacy_pool()
if last_scrape is None:
ls = self.get_kv('last_scrape')
if ls:
try:
last_scrape = datetime.strptime(ls, TIME_FORMAT)
except ValueError:
pass
self.log.debug('Last scrape %s', last_scrape)
self.check_health()
now = datetime.utcnow()
if not last_scrape:
next_scrape = now
else:
# align to scrape interval
scrape_frequency = self.scrape_frequency or 86400
seconds = (last_scrape -
datetime.utcfromtimestamp(0)).total_seconds()
seconds -= seconds % scrape_frequency
seconds += scrape_frequency
next_scrape = datetime.utcfromtimestamp(seconds)
if last_scrape:
self.log.debug('Last scrape %s, next scrape due %s',
last_scrape.strftime(TIME_FORMAT),
next_scrape.strftime(TIME_FORMAT))
else:
self.log.debug('Last scrape never, next scrape due %s',
next_scrape.strftime(TIME_FORMAT))
if now >= next_scrape:
self.scrape_all()
self.predict_all_devices()
last_scrape = now
self.set_kv('last_scrape',
last_scrape.strftime(TIME_FORMAT))
# sleep
sleep_interval = self.sleep_interval or 60
if not finished_loading_legacy:
sleep_interval = 2
self.log.debug('Sleeping for %d seconds', sleep_interval)
self.event.wait(sleep_interval)
self.event.clear()
def shutdown(self) -> None:
self.log.info('Stopping')
self.run = False
self.event.set()
def scrape_daemon(self, daemon_type: str,
daemon_id: str) -> Tuple[int, str, str]:
if not self.db_ready():
return -errno.EAGAIN, "", "mgr db not yet available"
raw_smart_data = self.do_scrape_daemon(daemon_type, daemon_id)
if raw_smart_data:
for device, raw_data in raw_smart_data.items():
data = self.extract_smart_features(raw_data)
if device and data:
self.put_device_metrics(device, data)
return 0, "", ""
def scrape_all(self) -> Tuple[int, str, str]:
if not self.db_ready():
return -errno.EAGAIN, "", "mgr db not yet available"
osdmap = self.get("osd_map")
assert osdmap is not None
did_device = {}
ids = []
for osd in osdmap['osds']:
ids.append(('osd', str(osd['osd'])))
monmap = self.get("mon_map")
for mon in monmap['mons']:
ids.append(('mon', mon['name']))
for daemon_type, daemon_id in ids:
raw_smart_data = self.do_scrape_daemon(daemon_type, daemon_id)
if not raw_smart_data:
continue
for device, raw_data in raw_smart_data.items():
if device in did_device:
self.log.debug('skipping duplicate %s' % device)
continue
did_device[device] = 1
data = self.extract_smart_features(raw_data)
if device and data:
self.put_device_metrics(device, data)
return 0, "", ""
def scrape_device(self, devid: str) -> Tuple[int, str, str]:
if not self.db_ready():
return -errno.EAGAIN, "", "mgr db not yet available"
r = self.get("device " + devid)
if not r or 'device' not in r.keys():
return -errno.ENOENT, '', 'device ' + devid + ' not found'
daemons = r['device'].get('daemons', [])
if not daemons:
return (-errno.EAGAIN, '',
'device ' + devid + ' not claimed by any active daemons')
(daemon_type, daemon_id) = daemons[0].split('.')
raw_smart_data = self.do_scrape_daemon(daemon_type,
daemon_id,
devid=devid)
if raw_smart_data:
for device, raw_data in raw_smart_data.items():
data = self.extract_smart_features(raw_data)
if device and data:
self.put_device_metrics(device, data)
return 0, "", ""
def do_scrape_daemon(self,
daemon_type: str,
daemon_id: str,
devid: str = '') -> Optional[Dict[str, Any]]:
"""
:return: a dict, or None if the scrape failed.
"""
self.log.debug('do_scrape_daemon %s.%s' % (daemon_type, daemon_id))
result = CommandResult('')
self.send_command(
result, daemon_type, daemon_id,
json.dumps({
'prefix': 'smart',
'format': 'json',
'devid': devid,
}), '')
r, outb, outs = result.wait()
try:
return json.loads(outb)
except (IndexError, ValueError):
self.log.error(
"Fail to parse JSON result from daemon {0}.{1} ({2})".format(
daemon_type, daemon_id, outb))
return None
def _prune_device_metrics(self) -> None:
SQL = """
DELETE FROM DeviceHealthMetrics
WHERE time < (strftime('%s', 'now') - ?);
"""
cursor = self.db.execute(SQL, (self.retention_period, ))
if cursor.rowcount >= 1:
self.log.info(f"pruned {cursor.rowcount} metrics")
def _create_device(self, devid: str) -> None:
SQL = """
INSERT OR IGNORE INTO Device VALUES (?);
"""
cursor = self.db.execute(SQL, (devid, ))
if cursor.rowcount >= 1:
self.log.info(f"created device {devid}")
else:
self.log.debug(f"device {devid} already exists")
def put_device_metrics(self, devid: str, data: Any) -> None:
SQL = """
INSERT INTO DeviceHealthMetrics (devid, raw_smart)
VALUES (?, ?);
"""
with self._db_lock, self.db:
self._create_device(devid)
self.db.execute(SQL, (devid, json.dumps(data)))
self._prune_device_metrics()
# extract wear level?
wear_level = get_ata_wear_level(data)
if wear_level is None:
wear_level = get_nvme_wear_level(data)
dev_data = self.get(f"device {devid}") or {}
if wear_level is not None:
if dev_data.get(wear_level) != str(wear_level):
dev_data["wear_level"] = str(wear_level)
self.log.debug(f"updating {devid} wear level to {wear_level}")
self.set_device_wear_level(devid, wear_level)
else:
if "wear_level" in dev_data:
del dev_data["wear_level"]
self.log.debug(f"removing {devid} wear level")
self.set_device_wear_level(devid, -1.0)
def _t2epoch(self, t: Optional[str]) -> int:
if t is None:
return 0
else:
return int(datetime.strptime(t, TIME_FORMAT).strftime("%s"))
def _get_device_metrics(
self,
devid: str,
sample: Optional[str] = None,
min_sample: Optional[str] = None) -> Dict[str, Dict[str, Any]]:
res = {}
SQL = """
SELECT time, raw_smart
FROM DeviceHealthMetrics
WHERE devid = ? AND (time = ? OR ? <= time)
ORDER BY time DESC;
"""
isample = self._t2epoch(sample)
imin_sample = self._t2epoch(min_sample)
self.log.debug(f"_get_device_metrics: {devid} {sample} {min_sample}")
with self._db_lock, self.db:
cursor = self.db.execute(SQL, (devid, isample, imin_sample))
for row in cursor:
t = row['time']
dt = datetime.utcfromtimestamp(t).strftime(TIME_FORMAT)
try:
res[dt] = json.loads(row['raw_smart'])
except (ValueError, IndexError):
self.log.debug(f"unable to parse value for {devid}:{t}")
pass
return res
def show_device_metrics(self, devid: str,
sample: Optional[str]) -> Tuple[int, str, str]:
# verify device exists
r = self.get("device " + devid)
if not r or 'device' not in r.keys():
return -errno.ENOENT, '', 'device ' + devid + ' not found'
# fetch metrics
res = self._get_device_metrics(devid, sample=sample)
return 0, json.dumps(res, indent=4, sort_keys=True), ''
def check_health(self) -> Tuple[int, str, str]:
self.log.info('Check health')
config = self.get('config')
min_in_ratio = float(config.get('mon_osd_min_in_ratio'))
mark_out_threshold_td = timedelta(seconds=self.mark_out_threshold)
warn_threshold_td = timedelta(seconds=self.warn_threshold)
checks: Dict[str, Dict[str, Union[int, str, Sequence[str]]]] = {}
health_warnings: Dict[str, List[str]] = {
DEVICE_HEALTH: [],
DEVICE_HEALTH_IN_USE: [],
}
devs = self.get("devices")
osds_in = {}
osds_out = {}
now = datetime.utcnow()
osdmap = self.get("osd_map")
assert osdmap is not None
for dev in devs['devices']:
if 'life_expectancy_max' not in dev:
continue
# ignore devices that are not consumed by any daemons
if not dev['daemons']:
continue
if not dev['life_expectancy_max'] or \
dev['life_expectancy_max'] == '0.000000':
continue
# life_expectancy_(min/max) is in the format of:
# '%Y-%m-%dT%H:%M:%S.%f%z', e.g.:
# '2019-01-20T21:12:12.000000Z'
life_expectancy_max = datetime.strptime(dev['life_expectancy_max'],
'%Y-%m-%dT%H:%M:%S.%f%z')
self.log.debug('device %s expectancy max %s', dev,
life_expectancy_max)
if life_expectancy_max - now <= mark_out_threshold_td:
if self.self_heal:
# dev['daemons'] == ["osd.0","osd.1","osd.2"]
if dev['daemons']:
osds = [
x for x in dev['daemons'] if x.startswith('osd.')
]
osd_ids = map(lambda x: x[4:], osds)
for _id in osd_ids:
if self.is_osd_in(osdmap, _id):
osds_in[_id] = life_expectancy_max
else:
osds_out[_id] = 1
if life_expectancy_max - now <= warn_threshold_td:
# device can appear in more than one location in case
# of SCSI multipath
device_locations = map(lambda x: x['host'] + ':' + x['dev'],
dev['location'])
health_warnings[DEVICE_HEALTH].append(
'%s (%s); daemons %s; life expectancy between %s and %s' %
(dev['devid'], ','.join(device_locations), ','.join(
dev.get('daemons',
['none'])), dev['life_expectancy_max'],
dev.get('life_expectancy_max', 'unknown')))
# OSD might be marked 'out' (which means it has no
# data), however PGs are still attached to it.
for _id in osds_out:
num_pgs = self.get_osd_num_pgs(_id)
if num_pgs > 0:
health_warnings[DEVICE_HEALTH_IN_USE].append(
'osd.%s is marked out '
'but still has %s PG(s)' % (_id, num_pgs))
if osds_in:
self.log.debug('osds_in %s' % osds_in)
# calculate target in ratio
num_osds = len(osdmap['osds'])
num_in = len([x for x in osdmap['osds'] if x['in']])
num_bad = len(osds_in)
# sort with next-to-fail first
bad_osds = sorted(osds_in.items(), key=operator.itemgetter(1))
did = 0
to_mark_out = []
for osd_id, when in bad_osds:
ratio = float(num_in - did - 1) / float(num_osds)
if ratio < min_in_ratio:
final_ratio = float(num_in - num_bad) / float(num_osds)
checks[DEVICE_HEALTH_TOOMANY] = {
'severity':
'warning',
'summary':
HEALTH_MESSAGES[DEVICE_HEALTH_TOOMANY],
'detail': [
'%d OSDs with failing device(s) would bring "in" ratio to %f < mon_osd_min_in_ratio %f'
% (num_bad - did, final_ratio, min_in_ratio)
]
}
break
to_mark_out.append(osd_id)
did += 1
if to_mark_out:
self.mark_out_etc(to_mark_out)
for warning, ls in health_warnings.items():
n = len(ls)
if n:
checks[warning] = {
'severity': 'warning',
'summary': HEALTH_MESSAGES[warning] % n,
'count': len(ls),
'detail': ls,
}
self.set_health_checks(checks)
return 0, "", ""
def is_osd_in(self, osdmap: Dict[str, Any], osd_id: str) -> bool:
for osd in osdmap['osds']:
if osd_id == str(osd['osd']):
return bool(osd['in'])
return False
def get_osd_num_pgs(self, osd_id: str) -> int:
stats = self.get('osd_stats')
assert stats is not None
for stat in stats['osd_stats']:
if osd_id == str(stat['osd']):
return stat['num_pgs']
return -1
def mark_out_etc(self, osd_ids: List[str]) -> None:
self.log.info('Marking out OSDs: %s' % osd_ids)
result = CommandResult('')
self.send_command(
result, 'mon', '',
json.dumps({
'prefix': 'osd out',
'format': 'json',
'ids': osd_ids,
}), '')
r, outb, outs = result.wait()
if r != 0:
self.log.warning(
'Could not mark OSD %s out. r: [%s], outb: [%s], outs: [%s]',
osd_ids, r, outb, outs)
for osd_id in osd_ids:
result = CommandResult('')
self.send_command(
result, 'mon', '',
json.dumps({
'prefix': 'osd primary-affinity',
'format': 'json',
'id': int(osd_id),
'weight': 0.0,
}), '')
r, outb, outs = result.wait()
if r != 0:
self.log.warning(
'Could not set osd.%s primary-affinity, '
'r: [%s], outb: [%s], outs: [%s]', osd_id, r, outb, outs)
def extract_smart_features(self, raw: Any) -> Any:
# FIXME: extract and normalize raw smartctl --json output and
# generate a dict of the fields we care about.
return raw
def predict_lift_expectancy(self, devid: str) -> Tuple[int, str, str]:
plugin_name = ''
model = self.get_ceph_option('device_failure_prediction_mode')
if cast(str, model).lower() == 'local':
plugin_name = 'diskprediction_local'
else:
return -1, '', 'unable to enable any disk prediction model[local/cloud]'
try:
can_run, _ = self.remote(plugin_name, 'can_run')
if can_run:
return self.remote(plugin_name,
'predict_life_expectancy',
devid=devid)
else:
return -1, '', f'{plugin_name} is not available'
except Exception:
return -1, '', 'unable to invoke diskprediction local or remote plugin'
def predict_all_devices(self) -> Tuple[int, str, str]:
plugin_name = ''
model = self.get_ceph_option('device_failure_prediction_mode')
if cast(str, model).lower() == 'local':
plugin_name = 'diskprediction_local'
else:
return -1, '', 'unable to enable any disk prediction model[local/cloud]'
try:
can_run, _ = self.remote(plugin_name, 'can_run')
if can_run:
return self.remote(plugin_name, 'predict_all_devices')
else:
return -1, '', f'{plugin_name} is not available'
except Exception:
return -1, '', 'unable to invoke diskprediction local or remote plugin'
def get_recent_device_metrics(
self, devid: str, min_sample: str) -> Dict[str, Dict[str, Any]]:
return self._get_device_metrics(devid, min_sample=min_sample)
def get_time_format(self) -> str:
return TIME_FORMAT
class Module(MgrModule, orchestrator.Orchestrator):
"""An Orchestrator that uses <Ansible Runner Service> to perform operations
"""
MODULE_OPTIONS = [
# url:port of the Ansible Runner Service
Option(name="server_location", type="str", default=""),
# Check server identity (True by default)
Option(name="verify_server", type="bool", default=True),
# Path to an alternative CA bundle
Option(name="ca_bundle", type="str", default="")
]
def __init__(self, *args, **kwargs):
super(Module, self).__init__(*args, **kwargs)
self.run = False
self.all_completions = []
self.ar_client = None
# TLS certificate and key file names used to connect with the external
# Ansible Runner Service
self.client_cert_fname = ""
self.client_key_fname = ""
# used to provide more verbose explanation of errors in status method
self.status_message = ""
def available(self):
""" Check if Ansible Runner service is working
"""
available = False
msg = ""
try:
if self.ar_client:
available = self.ar_client.is_operative()
if not available:
msg = "No response from Ansible Runner Service"
else:
msg = "Not possible to initialize connection with Ansible "\
"Runner service."
except AnsibleRunnerServiceError as ex:
available = False
msg = str(ex)
# Add more details to the detected problem
if self.status_message:
msg = "{}:\n{}".format(msg, self.status_message)
return (available, msg)
def wait(self, completions):
"""Given a list of Completion instances, progress any which are
incomplete.
:param completions: list of Completion instances
:Returns : True if everything is done.
"""
# Check progress and update status in each operation
# Access completion.status property do the trick
for operation in completions:
self.log.info("<%s> status:%s", operation, operation.status)
completions = filter(lambda x: not x.is_complete, completions)
ops_pending = len(completions)
self.log.info("Operations pending: %s", ops_pending)
return ops_pending == 0
def serve(self):
""" Mandatory for standby modules
"""
self.log.info("Starting Ansible Orchestrator module ...")
try:
# Verify config options and client certificates
self.verify_config()
# Ansible runner service client
self.ar_client = Client(
server_url=self.get_module_option('server_location', ''),
verify_server=self.get_module_option('verify_server', True),
ca_bundle=self.get_module_option('ca_bundle', ''),
client_cert=self.client_cert_fname,
client_key=self.client_key_fname,
logger=self.log)
except AnsibleRunnerServiceError:
self.log.exception(
"Ansible Runner Service not available. "
"Check external server status/TLS identity or "
"connection options. If configuration options changed"
" try to disable/enable the module.")
self.shutdown()
return
self.run = True
def shutdown(self):
self.log.info('Stopping Ansible orchestrator module')
self.run = False
def get_inventory(self, node_filter=None, refresh=False):
"""
:param : node_filter instance
:param : refresh any cached state
:Return : A AnsibleReadOperation instance (Completion Object)
"""
# Create a new read completion object for execute the playbook
playbook_operation = PlaybookOperation(
client=self.ar_client,
playbook=GET_STORAGE_DEVICES_CATALOG_PLAYBOOK,
logger=self.log,
result_pattern="list storage inventory",
params={})
# Assign the process_output function
playbook_operation.output_wizard = ProcessInventory(
self.ar_client, self.log)
playbook_operation.event_filter_list = ["runner_on_ok"]
# Execute the playbook to obtain data
self._launch_operation(playbook_operation)
return playbook_operation
def create_osds(self, drive_group, all_hosts):
"""Create one or more OSDs within a single Drive Group.
If no host provided the operation affects all the host in the OSDS role
:param drive_group: (orchestrator.DriveGroupSpec),
Drive group with the specification of drives to use
:param all_hosts : (List[str]),
List of hosts where the OSD's must be created
"""
# Transform drive group specification to Ansible playbook parameters
host, osd_spec = dg_2_ansible(drive_group)
# Create a new read completion object for execute the playbook
playbook_operation = PlaybookOperation(client=self.ar_client,
playbook=ADD_OSD_PLAYBOOK,
logger=self.log,
result_pattern="",
params=osd_spec,
querystr_dict={"limit": host})
# Filter to get the result
playbook_operation.output_wizard = ProcessPlaybookResult(
self.ar_client, self.log)
playbook_operation.event_filter_list = ["playbook_on_stats"]
# Execute the playbook
self._launch_operation(playbook_operation)
return playbook_operation
def remove_osds(self, osd_ids):
"""Remove osd's.
:param osd_ids: List of osd's to be removed (List[int])
"""
extravars = {
'osd_to_kill': ",".join([str(osd_id) for osd_id in osd_ids]),
'ireallymeanit': 'yes'
}
# Create a new read completion object for execute the playbook
playbook_operation = PlaybookOperation(client=self.ar_client,
playbook=REMOVE_OSD_PLAYBOOK,
logger=self.log,
result_pattern="",
params=extravars)
# Filter to get the result
playbook_operation.output_wizard = ProcessPlaybookResult(
self.ar_client, self.log)
playbook_operation.event_filter_list = ["playbook_on_stats"]
# Execute the playbook
self._launch_operation(playbook_operation)
return playbook_operation
def get_hosts(self):
"""Provides a list Inventory nodes
"""
host_ls_op = ARSOperation(self.ar_client, self.log, URL_GET_HOSTS)
host_ls_op.output_wizard = ProcessHostsList(self.ar_client, self.log)
return host_ls_op
def add_host(self, host):
"""
Add a host to the Ansible Runner Service inventory in the "orchestrator"
group
:param host: hostname
:returns : orchestrator.WriteCompletion
"""
url_group = URL_MANAGE_GROUP.format(group_name=ORCHESTRATOR_GROUP)
try:
# Create the orchestrator default group if not exist.
# If exists we ignore the error response
dummy_response = self.ar_client.http_post(url_group, "", {})
# Here, the default group exists so...
# Prepare the operation for adding the new host
add_url = URL_ADD_RM_HOSTS.format(
host_name=host, inventory_group=ORCHESTRATOR_GROUP)
operations = [HttpOperation(add_url, "post", "", None)]
except AnsibleRunnerServiceError as ex:
# Problems with the external orchestrator.
# Prepare the operation to return the error in a Completion object.
self.log.exception("Error checking <orchestrator> group: %s",
str(ex))
operations = [HttpOperation(url_group, "post", "", None)]
return ARSChangeOperation(self.ar_client, self.log, operations)
def remove_host(self, host):
"""
Remove a host from all the groups in the Ansible Runner Service
inventory.
:param host: hostname
:returns : orchestrator.WriteCompletion
"""
operations = []
host_groups = []
try:
# Get the list of groups where the host is included
groups_url = URL_GET_HOST_GROUPS.format(host_name=host)
response = self.ar_client.http_get(groups_url)
if response.status_code == requests.codes.ok:
host_groups = json.loads(response.text)["data"]["groups"]
except AnsibleRunnerServiceError:
self.log.exception("Error retrieving host groups")
if not host_groups:
# Error retrieving the groups, prepare the completion object to
# execute the problematic operation just to provide the error
# to the caller
operations = [HttpOperation(groups_url, "get")]
else:
# Build the operations list
operations = list(
map(
lambda x: HttpOperation(
URL_ADD_RM_HOSTS.format(host_name=host,
inventory_group=x), "delete"),
host_groups))
return ARSChangeOperation(self.ar_client, self.log, operations)
def _launch_operation(self, ansible_operation):
"""Launch the operation and add the operation to the completion objects
ongoing
:ansible_operation: A read/write ansible operation (completion object)
"""
# Execute the playbook
ansible_operation.execute_playbook()
# Add the operation to the list of things ongoing
self.all_completions.append(ansible_operation)
def verify_config(self):
"""Verify mandatory settings for the module and provide help to
configure properly the orchestrator
"""
the_crt = None
the_key = None
# Retrieve TLS content to use and check them
# First try to get certiticate and key content for this manager instance
# ex: mgr/ansible/mgr0/[crt/key]
self.log.info("Tying to use configured specific certificate and key"
"files for this server")
the_crt = self.get_store("{}/{}".format(self.get_mgr_id(), "crt"))
the_key = self.get_store("{}/{}".format(self.get_mgr_id(), "key"))
if the_crt is None or the_key is None:
# If not possible... try to get generic certificates and key content
# ex: mgr/ansible/[crt/key]
self.log.warning("Specific tls files for this manager not "\
"configured, trying to use generic files")
the_crt = self.get_store("crt")
the_key = self.get_store("key")
if the_crt is None or the_key is None:
self.status_message = "No client certificate configured. Please "\
"set Ansible Runner Service client "\
"certificate and key:\n"\
"ceph ansible set-ssl-certificate-"\
"{key,certificate} -i <file>"
self.log.error(self.status_message)
return
# generate certificate temp files
self.client_cert_fname = generate_temp_file("crt", the_crt)
self.client_key_fname = generate_temp_file("key", the_key)
self.status_message = verify_tls_files(self.client_cert_fname,
self.client_key_fname)
if self.status_message:
self.log.error(self.status_message)
return
# Check module options
if not self.get_module_option("server_location", ""):
self.status_message = "No Ansible Runner Service base URL "\
"<server_name>:<port>."\
"Try 'ceph config set mgr mgr/{0}/server_location "\
"<server name/ip>:<port>'".format(self.module_name)
self.log.error(self.status_message)
return
if self.get_module_option("verify_server", True):
self.status_message = "TLS server identity verification is enabled"\
" by default.Use 'ceph config set mgr mgr/{0}/verify_server False'"\
"to disable it.Use 'ceph config set mgr mgr/{0}/ca_bundle <path>'"\
"to point an alternative CA bundle path used for TLS server "\
"verification".format(self.module_name)
self.log.error(self.status_message)
return
# Everything ok
self.status_message = ""
#---------------------------------------------------------------------------
# Ansible Orchestrator self-owned commands
#---------------------------------------------------------------------------
@CLIWriteCommand("ansible set-ssl-certificate",
"name=mgr_id,type=CephString,req=false")
def set_tls_certificate(self, mgr_id=None, inbuf=None):
"""Load tls certificate in mon k-v store
"""
if inbuf is None:
return -errno.EINVAL, \
'Please specify the certificate file with "-i" option', ''
if mgr_id is not None:
self.set_store("{}/crt".format(mgr_id), inbuf)
else:
self.set_store("crt", inbuf)
return 0, "SSL certificate updated", ""
@CLIWriteCommand("ansible set-ssl-certificate-key",
"name=mgr_id,type=CephString,req=false")
def set_tls_certificate_key(self, mgr_id=None, inbuf=None):
"""Load tls certificate key in mon k-v store
"""
if inbuf is None:
return -errno.EINVAL, \
'Please specify the certificate key file with "-i" option', \
''
if mgr_id is not None:
self.set_store("{}/key".format(mgr_id), inbuf)
else:
self.set_store("key", inbuf)
return 0, "SSL certificate key updated", ""
class Hello(MgrModule):
# These are module options we understand. These can be set with
#
# ceph config set global mgr/hello/<name> <value>
#
# e.g.,
#
# ceph config set global mgr/hello/place Earth
#
MODULE_OPTIONS = [
Option(name='place',
default='world',
desc='a place in the world',
runtime=True), # can be updated at runtime (no mgr restart)
Option(name='emphatic',
type='bool',
desc='whether to say it loudly',
default=True,
runtime=True),
Option(name='foo',
type='str',
enum_allowed=['a', 'b', 'c'],
default='a',
runtime=True)
]
# These are "native" Ceph options that this module cares about.
NATIVE_OPTIONS = [
'mgr_tick_period',
]
def __init__(self, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
# set up some members to enable the serve() method and shutdown()
self.run = True
self.event = Event()
# ensure config options members are initialized; see config_notify()
self.config_notify()
# for mypy which does not run the code
if TYPE_CHECKING:
self.mgr_tick_period = 0
def config_notify(self) -> None:
"""
This method is called whenever one of our config options is changed.
"""
# This is some boilerplate that stores MODULE_OPTIONS in a class
# member, so that, for instance, the 'emphatic' option is always
# available as 'self.emphatic'.
for opt in self.MODULE_OPTIONS:
setattr(self,
opt['name'],
self.get_module_option(opt['name']))
self.log.debug(' mgr option %s = %s',
opt['name'], getattr(self, opt['name']))
# Do the same for the native options.
for opt in self.NATIVE_OPTIONS:
setattr(self,
opt,
self.get_ceph_option(opt))
self.log.debug(' native option %s = %s', opt, getattr(self, opt))
# there are CLI commands we implement
@CLIReadCommand('hello')
def hello(self, person_name: Optional[str] = None) -> HandleCommandResult:
"""
Say hello
"""
if person_name is None:
who = cast(str, self.get_module_option('place'))
else:
who = person_name
fin = '!' if self.get_module_option('emphatic') else ''
return HandleCommandResult(stdout=f'Hello, {who}{fin}')
@CLIReadCommand('count')
def count(self, num: int) -> HandleCommandResult:
"""
Do some counting
"""
ret = 0
out = ''
err = ''
if num < 1:
err = 'That\'s too small a number'
ret = -errno.EINVAL
elif num > 10:
err = 'That\'s too big a number'
ret = -errno.EINVAL
else:
out = 'Hello, I am the count!\n'
out += ', '.join([str(x) for x in range(1, num + 1)]) + '!'
return HandleCommandResult(retval=ret,
stdout=out,
stderr=err)
def serve(self) -> None:
"""
This method is called by the mgr when the module starts and can be
used for any background activity.
"""
self.log.info("Starting")
while self.run:
# Do some useful background work here.
# Use mgr_tick_period (default: 2) here just to illustrate
# consuming native ceph options. Any real background work
# would presumably have some more appropriate frequency.
sleep_interval = self.mgr_tick_period
self.log.debug('Sleeping for %d seconds', sleep_interval)
self.event.wait(sleep_interval)
self.event.clear()
def shutdown(self) -> None:
"""
This method is called by the mgr when the module needs to shut
down (i.e., when the serve() function needs to exit).
"""
self.log.info('Stopping')
self.run = False
self.event.set()
class RookOrchestrator(MgrModule, orchestrator.Orchestrator):
"""
Writes are a two-phase thing, firstly sending
the write to the k8s API (fast) and then waiting
for the corresponding change to appear in the
Ceph cluster (slow)
Right now, we are calling the k8s API synchronously.
"""
MODULE_OPTIONS: List[Option] = [
# TODO: configure k8s API addr instead of assuming local
Option(
'storage_class',
type='str',
default='local',
desc='storage class name for LSO-discovered PVs',
),
Option(
'drive_group_interval',
type='float',
default=300.0,
desc=
'interval in seconds between re-application of applied drive_groups',
),
]
@staticmethod
def can_run() -> Tuple[bool, str]:
if not kubernetes_imported:
return False, "`kubernetes` python module not found"
if not RookEnv().api_version_match():
return False, "Rook version unsupported."
return True, ''
def available(self) -> Tuple[bool, str, Dict[str, Any]]:
if not kubernetes_imported:
return False, "`kubernetes` python module not found", {}
elif not self._rook_env.has_namespace():
return False, "ceph-mgr not running in Rook cluster", {}
try:
self.k8s.list_namespaced_pod(self._rook_env.namespace)
except ApiException as e:
return False, "Cannot reach Kubernetes API: {}".format(e), {}
else:
return True, "", {}
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(RookOrchestrator, self).__init__(*args, **kwargs)
self._initialized = threading.Event()
self._k8s_CoreV1_api: Optional[client.CoreV1Api] = None
self._k8s_BatchV1_api: Optional[client.BatchV1Api] = None
self._k8s_CustomObjects_api: Optional[client.CustomObjectsApi] = None
self._k8s_StorageV1_api: Optional[client.StorageV1Api] = None
self._rook_cluster: Optional[RookCluster] = None
self._rook_env = RookEnv()
self._k8s_AppsV1_api: Optional[client.AppsV1Api] = None
self.config_notify()
if TYPE_CHECKING:
self.storage_class = 'foo'
self.drive_group_interval = 10.0
self._load_drive_groups()
self._shutdown = threading.Event()
def config_notify(self) -> None:
"""
This method is called whenever one of our config options is changed.
TODO: this method should be moved into mgr_module.py
"""
for opt in self.MODULE_OPTIONS:
setattr(
self,
opt['name'], # type: ignore
self.get_module_option(opt['name'])) # type: ignore
self.log.debug(' mgr option %s = %s', opt['name'],
getattr(self, opt['name'])) # type: ignore
assert isinstance(self.storage_class, str)
assert isinstance(self.drive_group_interval, float)
if self._rook_cluster:
self._rook_cluster.storage_class = self.storage_class
def shutdown(self) -> None:
self._shutdown.set()
@property
def k8s(self):
# type: () -> client.CoreV1Api
self._initialized.wait()
assert self._k8s_CoreV1_api is not None
return self._k8s_CoreV1_api
@property
def rook_cluster(self):
# type: () -> RookCluster
self._initialized.wait()
assert self._rook_cluster is not None
return self._rook_cluster
def serve(self) -> None:
# For deployed clusters, we should always be running inside
# a Rook cluster. For development convenience, also support
# running outside (reading ~/.kube config)
if self._rook_env.has_namespace():
config.load_incluster_config()
else:
self.log.warning("DEVELOPMENT ONLY: Reading kube config from ~")
config.load_kube_config()
# So that I can do port forwarding from my workstation - jcsp
from kubernetes.client import configuration
configuration.verify_ssl = False
self._k8s_CoreV1_api = client.CoreV1Api()
self._k8s_BatchV1_api = client.BatchV1Api()
self._k8s_CustomObjects_api = client.CustomObjectsApi()
self._k8s_StorageV1_api = client.StorageV1Api()
self._k8s_AppsV1_api = client.AppsV1Api()
try:
# XXX mystery hack -- I need to do an API call from
# this context, or subsequent API usage from handle_command
# fails with SSLError('bad handshake'). Suspect some kind of
# thread context setup in SSL lib?
self._k8s_CoreV1_api.list_namespaced_pod(self._rook_env.namespace)
except ApiException:
# Ignore here to make self.available() fail with a proper error message
pass
assert isinstance(self.storage_class, str)
self._rook_cluster = RookCluster(self._k8s_CoreV1_api,
self._k8s_BatchV1_api,
self._k8s_CustomObjects_api,
self._k8s_StorageV1_api,
self._k8s_AppsV1_api, self._rook_env,
self.storage_class)
self._initialized.set()
self.config_notify()
while not self._shutdown.is_set():
self._apply_drivegroups(list(self._drive_group_map.values()))
self._shutdown.wait(self.drive_group_interval)
@handle_orch_error
def get_inventory(
self,
host_filter: Optional[orchestrator.InventoryFilter] = None,
refresh: bool = False) -> List[orchestrator.InventoryHost]:
host_list = None
if host_filter and host_filter.hosts:
# Explicit host list
host_list = host_filter.hosts
elif host_filter and host_filter.labels:
# TODO: query k8s API to resolve to host list, and pass
# it into RookCluster.get_discovered_devices
raise NotImplementedError()
discovered_devs = self.rook_cluster.get_discovered_devices(host_list)
result = []
for host_name, host_devs in discovered_devs.items():
devs = []
for d in host_devs:
devs.append(d)
result.append(
orchestrator.InventoryHost(host_name, inventory.Devices(devs)))
return result
@handle_orch_error
def get_hosts(self):
# type: () -> List[orchestrator.HostSpec]
return self.rook_cluster.get_hosts()
@handle_orch_error
def describe_service(
self,
service_type: Optional[str] = None,
service_name: Optional[str] = None,
refresh: bool = False) -> List[orchestrator.ServiceDescription]:
now = datetime_now()
# CephCluster
cl = self.rook_cluster.rook_api_get("cephclusters/{0}".format(
self.rook_cluster.rook_env.cluster_name))
self.log.debug('CephCluster %s' % cl)
image_name = cl['spec'].get('cephVersion', {}).get('image', None)
num_nodes = len(self.rook_cluster.get_node_names())
spec = {}
if service_type == 'mon' or service_type is None:
spec['mon'] = orchestrator.ServiceDescription(
spec=ServiceSpec(
'mon',
placement=PlacementSpec(count=cl['spec'].get(
'mon', {}).get('count', 1), ),
),
size=cl['spec'].get('mon', {}).get('count', 1),
container_image_name=image_name,
last_refresh=now,
)
if service_type == 'mgr' or service_type is None:
spec['mgr'] = orchestrator.ServiceDescription(
spec=ServiceSpec(
'mgr',
placement=PlacementSpec.from_string('count:1'),
),
size=1,
container_image_name=image_name,
last_refresh=now,
)
if (service_type == 'crash'
or service_type is None and not cl['spec'].get(
'crashCollector', {}).get('disable', False)):
spec['crash'] = orchestrator.ServiceDescription(
spec=ServiceSpec(
'crash',
placement=PlacementSpec.from_string('*'),
),
size=num_nodes,
container_image_name=image_name,
last_refresh=now,
)
if service_type == 'mds' or service_type is None:
# CephFilesystems
all_fs = self.rook_cluster.get_resource("cephfilesystems")
for fs in all_fs:
svc = 'mds.' + fs['metadata']['name']
if svc in spec:
continue
# FIXME: we are conflating active (+ standby) with count
active = fs['spec'].get('metadataServer',
{}).get('activeCount', 1)
total_mds = active
if fs['spec'].get('metadataServer',
{}).get('activeStandby', False):
total_mds = active * 2
spec[svc] = orchestrator.ServiceDescription(
spec=ServiceSpec(
service_type='mds',
service_id=fs['metadata']['name'],
placement=PlacementSpec(count=active),
),
size=total_mds,
container_image_name=image_name,
last_refresh=now,
)
if service_type == 'rgw' or service_type is None:
# CephObjectstores
all_zones = self.rook_cluster.get_resource("cephobjectstores")
for zone in all_zones:
svc = 'rgw.' + zone['metadata']['name']
if svc in spec:
continue
active = zone['spec']['gateway']['instances']
if 'securePort' in zone['spec']['gateway']:
ssl = True
port = zone['spec']['gateway']['securePort']
else:
ssl = False
port = zone['spec']['gateway']['port'] or 80
rgw_zone = zone['spec'].get('zone', {}).get('name') or None
spec[svc] = orchestrator.ServiceDescription(
spec=RGWSpec(
service_id=zone['metadata']['name'],
rgw_zone=rgw_zone,
ssl=ssl,
rgw_frontend_port=port,
placement=PlacementSpec(count=active),
),
size=active,
container_image_name=image_name,
last_refresh=now,
)
if service_type == 'nfs' or service_type is None:
# CephNFSes
all_nfs = self.rook_cluster.get_resource("cephnfses")
nfs_pods = self.rook_cluster.describe_pods('nfs', None, None)
for nfs in all_nfs:
if nfs['spec']['rados']['pool'] != NFS_POOL_NAME:
continue
nfs_name = nfs['metadata']['name']
svc = 'nfs.' + nfs_name
if svc in spec:
continue
active = nfs['spec'].get('server', {}).get('active')
creation_timestamp = datetime.datetime.strptime(
nfs['metadata']['creationTimestamp'], '%Y-%m-%dT%H:%M:%SZ')
spec[svc] = orchestrator.ServiceDescription(
spec=NFSServiceSpec(
service_id=nfs_name,
placement=PlacementSpec(count=active),
),
size=active,
last_refresh=now,
running=len([
1 for pod in nfs_pods
if pod['labels']['ceph_nfs'] == nfs_name
]),
created=creation_timestamp.astimezone(
tz=datetime.timezone.utc))
if service_type == 'osd' or service_type is None:
# OSDs
# FIXME: map running OSDs back to their respective services...
# the catch-all unmanaged
all_osds = self.rook_cluster.get_osds()
svc = 'osd'
spec[svc] = orchestrator.ServiceDescription(
spec=DriveGroupSpec(
unmanaged=True,
service_type='osd',
),
size=len(all_osds),
last_refresh=now,
running=sum(osd.status.phase == 'Running' for osd in all_osds))
# drivegroups
for name, dg in self._drive_group_map.items():
spec[f'osd.{name}'] = orchestrator.ServiceDescription(
spec=dg,
last_refresh=now,
size=0,
running=0,
)
if service_type == 'rbd-mirror' or service_type is None:
# rbd-mirrors
all_mirrors = self.rook_cluster.get_resource("cephrbdmirrors")
for mirror in all_mirrors:
logging.warn(mirror)
mirror_name = mirror['metadata']['name']
svc = 'rbd-mirror.' + mirror_name
if svc in spec:
continue
spec[svc] = orchestrator.ServiceDescription(
spec=ServiceSpec(
service_id=mirror_name,
service_type="rbd-mirror",
placement=PlacementSpec(count=1),
),
size=1,
last_refresh=now,
)
for dd in self._list_daemons():
if dd.service_name() not in spec:
continue
service = spec[dd.service_name()]
service.running += 1
if not service.container_image_id:
service.container_image_id = dd.container_image_id
if not service.container_image_name:
service.container_image_name = dd.container_image_name
if service.last_refresh is None or not dd.last_refresh or dd.last_refresh < service.last_refresh:
service.last_refresh = dd.last_refresh
if service.created is None or dd.created is None or dd.created < service.created:
service.created = dd.created
return [v for k, v in spec.items()]
@handle_orch_error
def list_daemons(
self,
service_name: Optional[str] = None,
daemon_type: Optional[str] = None,
daemon_id: Optional[str] = None,
host: Optional[str] = None,
refresh: bool = False) -> List[orchestrator.DaemonDescription]:
return self._list_daemons(service_name=service_name,
daemon_type=daemon_type,
daemon_id=daemon_id,
host=host,
refresh=refresh)
def _list_daemons(
self,
service_name: Optional[str] = None,
daemon_type: Optional[str] = None,
daemon_id: Optional[str] = None,
host: Optional[str] = None,
refresh: bool = False) -> List[orchestrator.DaemonDescription]:
pods = self.rook_cluster.describe_pods(daemon_type, daemon_id, host)
self.log.debug('pods %s' % pods)
result = []
for p in pods:
sd = orchestrator.DaemonDescription()
sd.hostname = p['hostname']
sd.daemon_type = p['labels']['app'].replace('rook-ceph-', '')
status = {
'Pending': orchestrator.DaemonDescriptionStatus.error,
'Running': orchestrator.DaemonDescriptionStatus.running,
'Succeeded': orchestrator.DaemonDescriptionStatus.stopped,
'Failed': orchestrator.DaemonDescriptionStatus.error,
'Unknown': orchestrator.DaemonDescriptionStatus.error,
}[p['phase']]
sd.status = status
sd.status_desc = p['phase']
if 'ceph_daemon_id' in p['labels']:
sd.daemon_id = p['labels']['ceph_daemon_id']
elif 'ceph-osd-id' in p['labels']:
sd.daemon_id = p['labels']['ceph-osd-id']
else:
# Unknown type -- skip it
continue
if service_name is not None and service_name != sd.service_name():
continue
sd.container_image_name = p['container_image_name']
sd.container_image_id = p['container_image_id']
sd.created = p['created']
sd.last_configured = p['created']
sd.last_deployed = p['created']
sd.started = p['started']
sd.last_refresh = p['refreshed']
result.append(sd)
return result
def _get_pool_params(self) -> Tuple[int, str]:
num_replicas = self.get_ceph_option('osd_pool_default_size')
assert type(num_replicas) is int
leaf_type_id = self.get_ceph_option('osd_crush_chooseleaf_type')
assert type(leaf_type_id) is int
crush = self.get('osd_map_crush')
leaf_type = 'host'
for t in crush['types']:
if t['type_id'] == leaf_type_id:
leaf_type = t['name']
break
return num_replicas, leaf_type
@handle_orch_error
def remove_service(self, service_name: str, force: bool = False) -> str:
if service_name == 'rbd-mirror':
return self.rook_cluster.rm_service('cephrbdmirrors',
'default-rbd-mirror')
service_type, service_id = service_name.split('.', 1)
if service_type == 'mds':
return self.rook_cluster.rm_service('cephfilesystems', service_id)
elif service_type == 'rgw':
return self.rook_cluster.rm_service('cephobjectstores', service_id)
elif service_type == 'nfs':
ret, out, err = self.mon_command({'prefix': 'auth ls'})
matches = re.findall(rf'client\.nfs-ganesha\.{service_id}\..*',
out)
for match in matches:
self.check_mon_command({'prefix': 'auth rm', 'entity': match})
return self.rook_cluster.rm_service('cephnfses', service_id)
elif service_type == 'rbd-mirror':
return self.rook_cluster.rm_service('cephrbdmirrors', service_id)
elif service_type == 'osd':
if service_id in self._drive_group_map:
del self._drive_group_map[service_id]
self._save_drive_groups()
return f'Removed {service_name}'
elif service_type == 'ingress':
self.log.info("{0} service '{1}' does not exist".format(
'ingress', service_id))
return 'The Rook orchestrator does not currently support ingress'
else:
raise orchestrator.OrchestratorError(
f'Service type {service_type} not supported')
def zap_device(self, host: str, path: str) -> OrchResult[str]:
try:
self.rook_cluster.create_zap_job(host, path)
except Exception as e:
logging.error(e)
return OrchResult(
None,
Exception("Unable to zap device: " +
str(e.with_traceback(None))))
return OrchResult(f'{path} on {host} zapped')
@handle_orch_error
def apply_mon(self, spec):
# type: (ServiceSpec) -> str
if spec.placement.hosts or spec.placement.label:
raise RuntimeError("Host list or label is not supported by rook.")
return self.rook_cluster.update_mon_count(spec.placement.count)
def apply_rbd_mirror(self, spec: ServiceSpec) -> OrchResult[str]:
try:
self.rook_cluster.rbd_mirror(spec)
return OrchResult("Success")
except Exception as e:
return OrchResult(None, e)
@handle_orch_error
def apply_mds(self, spec):
# type: (ServiceSpec) -> str
num_replicas, leaf_type = self._get_pool_params()
return self.rook_cluster.apply_filesystem(spec, num_replicas,
leaf_type)
@handle_orch_error
def apply_rgw(self, spec):
# type: (RGWSpec) -> str
num_replicas, leaf_type = self._get_pool_params()
return self.rook_cluster.apply_objectstore(spec, num_replicas,
leaf_type)
@handle_orch_error
def apply_nfs(self, spec):
# type: (NFSServiceSpec) -> str
try:
return self.rook_cluster.apply_nfsgw(spec, self)
except Exception as e:
logging.error(e)
return "Unable to create NFS daemon, check logs for more traceback\n" + str(
e.with_traceback(None))
@handle_orch_error
def remove_daemons(self, names: List[str]) -> List[str]:
return self.rook_cluster.remove_pods(names)
def apply_drivegroups(
self, specs: List[DriveGroupSpec]) -> OrchResult[List[str]]:
for drive_group in specs:
self._drive_group_map[str(drive_group.service_id)] = drive_group
self._save_drive_groups()
return OrchResult(self._apply_drivegroups(specs))
def _apply_drivegroups(self, ls: List[DriveGroupSpec]) -> List[str]:
all_hosts = raise_if_exception(self.get_hosts())
result_list: List[str] = []
for drive_group in ls:
matching_hosts = drive_group.placement.filter_matching_hosts(
lambda label=None, as_hostspec=None: all_hosts)
if not self.rook_cluster.node_exists(matching_hosts[0]):
raise RuntimeError("Node '{0}' is not in the Kubernetes "
"cluster".format(matching_hosts))
# Validate whether cluster CRD can accept individual OSD
# creations (i.e. not useAllDevices)
if not self.rook_cluster.can_create_osd():
raise RuntimeError("Rook cluster configuration does not "
"support OSD creation.")
result_list.append(
self.rook_cluster.add_osds(drive_group, matching_hosts))
return result_list
def _load_drive_groups(self) -> None:
stored_drive_group = self.get_store("drive_group_map")
self._drive_group_map: Dict[str, DriveGroupSpec] = {}
if stored_drive_group:
for name, dg in json.loads(stored_drive_group).items():
try:
self._drive_group_map[name] = DriveGroupSpec.from_json(dg)
except ValueError as e:
self.log.error(
f'Failed to load drive group {name} ({dg}): {e}')
def _save_drive_groups(self) -> None:
json_drive_group_map = {
name: dg.to_json()
for name, dg in self._drive_group_map.items()
}
self.set_store("drive_group_map", json.dumps(json_drive_group_map))
def remove_osds(self,
osd_ids: List[str],
replace: bool = False,
force: bool = False,
zap: bool = False) -> OrchResult[str]:
assert self._rook_cluster is not None
if zap:
raise RuntimeError(
"Rook does not support zapping devices during OSD removal.")
res = self._rook_cluster.remove_osds(osd_ids, replace, force,
self.mon_command)
return OrchResult(res)
def add_host_label(self, host: str, label: str) -> OrchResult[str]:
return self.rook_cluster.add_host_label(host, label)
def remove_host_label(self, host: str, label: str) -> OrchResult[str]:
return self.rook_cluster.remove_host_label(host, label)
"""
@handle_orch_error
def create_osds(self, drive_group):
# type: (DriveGroupSpec) -> str
# Creates OSDs from a drive group specification.
# $: ceph orch osd create -i <dg.file>
# The drivegroup file must only contain one spec at a time.
#
targets = [] # type: List[str]
if drive_group.data_devices and drive_group.data_devices.paths:
targets += [d.path for d in drive_group.data_devices.paths]
if drive_group.data_directories:
targets += drive_group.data_directories
all_hosts = raise_if_exception(self.get_hosts())
matching_hosts = drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: all_hosts)
assert len(matching_hosts) == 1
if not self.rook_cluster.node_exists(matching_hosts[0]):
raise RuntimeError("Node '{0}' is not in the Kubernetes "
"cluster".format(matching_hosts))
# Validate whether cluster CRD can accept individual OSD
# creations (i.e. not useAllDevices)
if not self.rook_cluster.can_create_osd():
raise RuntimeError("Rook cluster configuration does not "
"support OSD creation.")
return self.rook_cluster.add_osds(drive_group, matching_hosts)
# TODO: this was the code to update the progress reference:
@handle_orch_error
def has_osds(matching_hosts: List[str]) -> bool:
# Find OSD pods on this host
pod_osd_ids = set()
pods = self.k8s.list_namespaced_pod(self._rook_env.namespace,
label_selector="rook_cluster={},app=rook-ceph-osd".format(self._rook_env.cluster_name),
field_selector="spec.nodeName={0}".format(
matching_hosts[0]
)).items
for p in pods:
pod_osd_ids.add(int(p.metadata.labels['ceph-osd-id']))
self.log.debug('pod_osd_ids={0}'.format(pod_osd_ids))
found = []
osdmap = self.get("osd_map")
for osd in osdmap['osds']:
osd_id = osd['osd']
if osd_id not in pod_osd_ids:
continue
metadata = self.get_metadata('osd', "%s" % osd_id)
if metadata and metadata['devices'] in targets:
found.append(osd_id)
else:
self.log.info("ignoring osd {0} {1}".format(
osd_id, metadata['devices'] if metadata else 'DNE'
))
return found is not None
"""
@handle_orch_error
def blink_device_light(
self, ident_fault: str, on: bool,
locs: List[orchestrator.DeviceLightLoc]) -> List[str]:
return self.rook_cluster.blink_light(ident_fault, on, locs)
class Module(MgrModule):
MODULE_OPTIONS = [
Option(name='subtree',
type='str',
default='rack',
desc='CRUSH level for which to create a local pool',
runtime=True),
Option(name='failure_domain',
type='str',
default='host',
desc='failure domain for any created local pool',
runtime=True),
Option(name='min_size',
type='int',
desc='default min_size for any created local pool',
runtime=True),
Option(name='num_rep',
type='int',
default=3,
desc='default replica count for any created local pool',
runtime=True),
Option(name='pg_num',
type='int',
default=128,
desc='default pg_num for any created local pool',
runtime=True),
Option(name='prefix',
type='str',
default='',
desc='name prefix for any created local pool',
runtime=True),
]
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
self.serve_event = threading.Event()
def notify(self, notify_type: str, notify_id: str) -> None:
if notify_type == 'osd_map':
self.handle_osd_map()
def handle_osd_map(self) -> None:
"""
Check pools on each OSDMap change
"""
subtree_type = cast(str, self.get_module_option('subtree'))
failure_domain = self.get_module_option('failure_domain')
pg_num = self.get_module_option('pg_num')
num_rep = self.get_module_option('num_rep')
min_size = self.get_module_option('min_size')
prefix = cast(
str,
self.get_module_option('prefix')) or 'by-' + subtree_type + '-'
osdmap = self.get("osd_map")
lpools = []
for pool in osdmap['pools']:
if pool['pool_name'].find(prefix) == 0:
lpools.append(pool['pool_name'])
self.log.debug('localized pools = %s', lpools)
subtrees = []
tree = self.get('osd_map_tree')
for node in tree['nodes']:
if node['type'] == subtree_type:
subtrees.append(node['name'])
pool_name = prefix + node['name']
if pool_name not in lpools:
self.log.info('Creating localized pool %s', pool_name)
#
result = CommandResult("")
self.send_command(
result, "mon", "",
json.dumps({
"prefix": "osd crush rule create-replicated",
"format": "json",
"name": pool_name,
"root": node['name'],
"type": failure_domain,
}), "")
r, outb, outs = result.wait()
result = CommandResult("")
self.send_command(
result, "mon", "",
json.dumps({
"prefix": "osd pool create",
"format": "json",
"pool": pool_name,
'rule': pool_name,
"pool_type": 'replicated',
'pg_num': pg_num,
}), "")
r, outb, outs = result.wait()
result = CommandResult("")
self.send_command(
result, "mon", "",
json.dumps({
"prefix": "osd pool set",
"format": "json",
"pool": pool_name,
'var': 'size',
"val": str(num_rep),
}), "")
r, outb, outs = result.wait()
if min_size:
result = CommandResult("")
self.send_command(
result, "mon", "",
json.dumps({
"prefix": "osd pool set",
"format": "json",
"pool": pool_name,
'var': 'min_size',
"val": str(min_size),
}), "")
r, outb, outs = result.wait()
# TODO remove pools for hosts that don't exist?
def serve(self) -> None:
self.handle_osd_map()
self.serve_event.wait()
self.serve_event.clear()
def shutdown(self) -> None:
self.serve_event.set()
class Module(MgrModule):
metadata_keys = [
"arch", "ceph_version", "os", "cpu", "kernel_description",
"kernel_version", "distro_description", "distro"
]
MODULE_OPTIONS = [
Option(name='url',
type='str',
default='https://telemetry.ceph.com/report'),
Option(name='device_url',
type='str',
default='https://telemetry.ceph.com/device'),
Option(name='enabled', type='bool', default=False),
Option(name='last_opt_revision', type='int', default=1),
Option(name='leaderboard', type='bool', default=False),
Option(name='description', type='str', default=None),
Option(name='contact', type='str', default=None),
Option(name='organization', type='str', default=None),
Option(name='proxy', type='str', default=None),
Option(name='interval', type='int', default=24, min=8),
Option(name='channel_basic',
type='bool',
default=True,
desc='Share basic cluster information (size, version)'),
Option(
name='channel_ident',
type='bool',
default=False,
desc=
'Share a user-provided description and/or contact email for the cluster'
),
Option(
name='channel_crash',
type='bool',
default=True,
desc=
'Share metadata about Ceph daemon crashes (version, stack straces, etc)'
),
Option(
name='channel_device',
type='bool',
default=True,
desc=
('Share device health metrics '
'(e.g., SMART data, minus potentially identifying info like serial numbers)'
)),
]
@property
def config_keys(self) -> Dict[str, OptionValue]:
return dict(
(o['name'], o.get('default', None)) for o in self.MODULE_OPTIONS)
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
self.event = Event()
self.run = False
self.last_upload: Optional[int] = None
self.last_report: Dict[str, Any] = dict()
self.report_id: Optional[str] = None
self.salt: Optional[str] = None
# for mypy which does not run the code
if TYPE_CHECKING:
self.url = ''
self.device_url = ''
self.enabled = False
self.last_opt_revision = 0
self.leaderboard = ''
self.interval = 0
self.proxy = ''
self.channel_basic = True
self.channel_ident = False
self.channel_crash = True
self.channel_device = True
def config_notify(self) -> None:
for opt in self.MODULE_OPTIONS:
setattr(self, opt['name'], self.get_module_option(opt['name']))
self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name']))
# wake up serve() thread
self.event.set()
def load(self) -> None:
last_upload = self.get_store('last_upload', None)
if last_upload is None:
self.last_upload = None
else:
self.last_upload = int(last_upload)
report_id = self.get_store('report_id', None)
if report_id is None:
self.report_id = str(uuid.uuid4())
self.set_store('report_id', self.report_id)
else:
self.report_id = report_id
salt = self.get_store('salt', None)
if salt is None:
self.salt = str(uuid.uuid4())
self.set_store('salt', self.salt)
else:
self.salt = salt
def gather_osd_metadata(
self, osd_map: Dict[str,
List[Dict[str,
int]]]) -> Dict[str, Dict[str, int]]:
keys = ["osd_objectstore", "rotational"]
keys += self.metadata_keys
metadata: Dict[str, Dict[str, int]] = dict()
for key in keys:
metadata[key] = defaultdict(int)
for osd in osd_map['osds']:
res = self.get_metadata('osd', str(osd['osd']))
if res is None:
self.log.debug('Could not get metadata for osd.%s' %
str(osd['osd']))
continue
for k, v in res.items():
if k not in keys:
continue
metadata[k][v] += 1
return metadata
def gather_mon_metadata(
self, mon_map: Dict[str,
List[Dict[str,
str]]]) -> Dict[str, Dict[str, int]]:
keys = list()
keys += self.metadata_keys
metadata: Dict[str, Dict[str, int]] = dict()
for key in keys:
metadata[key] = defaultdict(int)
for mon in mon_map['mons']:
res = self.get_metadata('mon', mon['name'])
if res is None:
self.log.debug('Could not get metadata for mon.%s' %
(mon['name']))
continue
for k, v in res.items():
if k not in keys:
continue
metadata[k][v] += 1
return metadata
def gather_crush_info(
self
) -> Dict[str, Union[int, bool, List[int], Dict[str, int], Dict[int,
int]]]:
osdmap = self.get_osdmap()
crush_raw = osdmap.get_crush()
crush = crush_raw.dump()
BucketKeyT = TypeVar('BucketKeyT', int, str)
def inc(d: Dict[BucketKeyT, int], k: BucketKeyT) -> None:
if k in d:
d[k] += 1
else:
d[k] = 1
device_classes: Dict[str, int] = {}
for dev in crush['devices']:
inc(device_classes, dev.get('class', ''))
bucket_algs: Dict[str, int] = {}
bucket_types: Dict[str, int] = {}
bucket_sizes: Dict[int, int] = {}
for bucket in crush['buckets']:
if '~' in bucket['name']: # ignore shadow buckets
continue
inc(bucket_algs, bucket['alg'])
inc(bucket_types, bucket['type_id'])
inc(bucket_sizes, len(bucket['items']))
return {
'num_devices': len(crush['devices']),
'num_types': len(crush['types']),
'num_buckets': len(crush['buckets']),
'num_rules': len(crush['rules']),
'device_classes': list(device_classes.values()),
'tunables': crush['tunables'],
'compat_weight_set': '-1' in crush['choose_args'],
'num_weight_sets': len(crush['choose_args']),
'bucket_algs': bucket_algs,
'bucket_sizes': bucket_sizes,
'bucket_types': bucket_types,
}
def gather_configs(self) -> Dict[str, List[str]]:
# cluster config options
cluster = set()
r, outb, outs = self.mon_command({
'prefix': 'config dump',
'format': 'json'
})
if r != 0:
return {}
try:
dump = json.loads(outb)
except json.decoder.JSONDecodeError:
return {}
for opt in dump:
name = opt.get('name')
if name:
cluster.add(name)
# daemon-reported options (which may include ceph.conf)
active = set()
ls = self.get("modified_config_options")
for opt in ls.get('options', {}):
active.add(opt)
return {
'cluster_changed': sorted(list(cluster)),
'active_changed': sorted(list(active)),
}
def gather_crashinfo(self) -> List[Dict[str, str]]:
crashlist: List[Dict[str, str]] = list()
errno, crashids, err = self.remote('crash', 'ls')
if errno:
return crashlist
for crashid in crashids.split():
errno, crashinfo, err = self.remote('crash', 'do_info', crashid)
if errno:
continue
c = json.loads(crashinfo)
# redact hostname
del c['utsname_hostname']
# entity_name might have more than one '.', beware
(etype, eid) = c.get('entity_name', '').split('.', 1)
m = hashlib.sha1()
assert self.salt
m.update(self.salt.encode('utf-8'))
m.update(eid.encode('utf-8'))
m.update(self.salt.encode('utf-8'))
c['entity_name'] = etype + '.' + m.hexdigest()
# redact final line of python tracebacks, as the exception
# payload may contain identifying information
if 'mgr_module' in c:
c['backtrace'][-1] = '<redacted>'
crashlist.append(c)
return crashlist
def get_active_channels(self) -> List[str]:
r = []
if self.channel_basic:
r.append('basic')
if self.channel_crash:
r.append('crash')
if self.channel_device:
r.append('device')
if self.channel_ident:
r.append('ident')
return r
def gather_device_report(self) -> Dict[str, Dict[str, Dict[str, str]]]:
try:
time_format = self.remote('devicehealth', 'get_time_format')
except Exception:
return {}
cutoff = datetime.utcnow() - timedelta(hours=self.interval * 2)
min_sample = cutoff.strftime(time_format)
devices = self.get('devices')['devices']
# anon-host-id -> anon-devid -> { timestamp -> record }
res: Dict[str, Dict[str, Dict[str, str]]] = {}
for d in devices:
devid = d['devid']
try:
# this is a map of stamp -> {device info}
m = self.remote('devicehealth', 'get_recent_device_metrics',
devid, min_sample)
except Exception:
continue
# anonymize host id
try:
host = d['location'][0]['host']
except KeyError:
continue
anon_host = self.get_store('host-id/%s' % host)
if not anon_host:
anon_host = str(uuid.uuid1())
self.set_store('host-id/%s' % host, anon_host)
serial = None
for dev, rep in m.items():
rep['host_id'] = anon_host
if serial is None and 'serial_number' in rep:
serial = rep['serial_number']
# anonymize device id
anon_devid = self.get_store('devid-id/%s' % devid)
if not anon_devid:
# ideally devid is 'vendor_model_serial',
# but can also be 'model_serial', 'serial'
if '_' in devid:
anon_devid = f"{devid.rsplit('_', 1)[0]}_{uuid.uuid1()}"
else:
anon_devid = str(uuid.uuid1())
self.set_store('devid-id/%s' % devid, anon_devid)
self.log.info('devid %s / %s, host %s / %s' %
(devid, anon_devid, host, anon_host))
# anonymize the smartctl report itself
if serial:
m_str = json.dumps(m)
m = json.loads(m_str.replace(serial, 'deleted'))
if anon_host not in res:
res[anon_host] = {}
res[anon_host][anon_devid] = m
return res
def get_latest(self, daemon_type: str, daemon_name: str, stat: str) -> int:
data = self.get_counter(daemon_type, daemon_name, stat)[stat]
if data:
return data[-1][1]
else:
return 0
def compile_report(self,
channels: Optional[List[str]] = None) -> Dict[str, Any]:
if not channels:
channels = self.get_active_channels()
report = {
'leaderboard': self.leaderboard,
'report_version': 1,
'report_timestamp': datetime.utcnow().isoformat(),
'report_id': self.report_id,
'channels': channels,
'channels_available': ALL_CHANNELS,
'license': LICENSE,
}
if 'ident' in channels:
for option in ['description', 'contact', 'organization']:
report[option] = getattr(self, option)
if 'basic' in channels:
mon_map = self.get('mon_map')
osd_map = self.get('osd_map')
service_map = self.get('service_map')
fs_map = self.get('fs_map')
df = self.get('df')
report['created'] = mon_map['created']
# mons
v1_mons = 0
v2_mons = 0
ipv4_mons = 0
ipv6_mons = 0
for mon in mon_map['mons']:
for a in mon['public_addrs']['addrvec']:
if a['type'] == 'v2':
v2_mons += 1
elif a['type'] == 'v1':
v1_mons += 1
if a['addr'].startswith('['):
ipv6_mons += 1
else:
ipv4_mons += 1
report['mon'] = {
'count': len(mon_map['mons']),
'features': mon_map['features'],
'min_mon_release': mon_map['min_mon_release'],
'v1_addr_mons': v1_mons,
'v2_addr_mons': v2_mons,
'ipv4_addr_mons': ipv4_mons,
'ipv6_addr_mons': ipv6_mons,
}
report['config'] = self.gather_configs()
# pools
rbd_num_pools = 0
rbd_num_images_by_pool = []
rbd_mirroring_by_pool = []
num_pg = 0
report['pools'] = list()
for pool in osd_map['pools']:
num_pg += pool['pg_num']
ec_profile = {}
if pool['erasure_code_profile']:
orig = osd_map['erasure_code_profiles'].get(
pool['erasure_code_profile'], {})
ec_profile = {
k: orig[k]
for k in orig.keys() if k in [
'k', 'm', 'plugin', 'technique',
'crush-failure-domain', 'l'
]
}
cast(List[Dict[str, Any]], report['pools']).append({
'pool':
pool['pool'],
'pg_num':
pool['pg_num'],
'pgp_num':
pool['pg_placement_num'],
'size':
pool['size'],
'min_size':
pool['min_size'],
'pg_autoscale_mode':
pool['pg_autoscale_mode'],
'target_max_bytes':
pool['target_max_bytes'],
'target_max_objects':
pool['target_max_objects'],
'type': ['', 'replicated', '', 'erasure'][pool['type']],
'erasure_code_profile':
ec_profile,
'cache_mode':
pool['cache_mode'],
})
if 'rbd' in pool['application_metadata']:
rbd_num_pools += 1
ioctx = self.rados.open_ioctx(pool['pool_name'])
rbd_num_images_by_pool.append(
sum(1 for _ in rbd.RBD().list2(ioctx)))
rbd_mirroring_by_pool.append(rbd.RBD().mirror_mode_get(
ioctx) != rbd.RBD_MIRROR_MODE_DISABLED)
report['rbd'] = {
'num_pools': rbd_num_pools,
'num_images_by_pool': rbd_num_images_by_pool,
'mirroring_by_pool': rbd_mirroring_by_pool
}
# osds
cluster_network = False
for osd in osd_map['osds']:
if osd['up'] and not cluster_network:
front_ip = osd['public_addrs']['addrvec'][0]['addr'].split(
':')[0]
back_ip = osd['cluster_addrs']['addrvec'][0]['addr'].split(
':')[0]
if front_ip != back_ip:
cluster_network = True
report['osd'] = {
'count': len(osd_map['osds']),
'require_osd_release': osd_map['require_osd_release'],
'require_min_compat_client':
osd_map['require_min_compat_client'],
'cluster_network': cluster_network,
}
# crush
report['crush'] = self.gather_crush_info()
# cephfs
report['fs'] = {
'count': len(fs_map['filesystems']),
'feature_flags': fs_map['feature_flags'],
'num_standby_mds': len(fs_map['standbys']),
'filesystems': [],
}
num_mds = len(fs_map['standbys'])
for fsm in fs_map['filesystems']:
fs = fsm['mdsmap']
num_sessions = 0
cached_ino = 0
cached_dn = 0
cached_cap = 0
subtrees = 0
rfiles = 0
rbytes = 0
rsnaps = 0
for gid, mds in fs['info'].items():
num_sessions += self.get_latest(
'mds', mds['name'], 'mds_sessions.session_count')
cached_ino += self.get_latest('mds', mds['name'],
'mds_mem.ino')
cached_dn += self.get_latest('mds', mds['name'],
'mds_mem.dn')
cached_cap += self.get_latest('mds', mds['name'],
'mds_mem.cap')
subtrees += self.get_latest('mds', mds['name'],
'mds.subtrees')
if mds['rank'] == 0:
rfiles = self.get_latest('mds', mds['name'],
'mds.root_rfiles')
rbytes = self.get_latest('mds', mds['name'],
'mds.root_rbytes')
rsnaps = self.get_latest('mds', mds['name'],
'mds.root_rsnaps')
report['fs']['filesystems'].append({ # type: ignore
'max_mds': fs['max_mds'],
'ever_allowed_features': fs['ever_allowed_features'],
'explicitly_allowed_features': fs['explicitly_allowed_features'],
'num_in': len(fs['in']),
'num_up': len(fs['up']),
'num_standby_replay': len(
[mds for gid, mds in fs['info'].items()
if mds['state'] == 'up:standby-replay']),
'num_mds': len(fs['info']),
'num_sessions': num_sessions,
'cached_inos': cached_ino,
'cached_dns': cached_dn,
'cached_caps': cached_cap,
'cached_subtrees': subtrees,
'balancer_enabled': len(fs['balancer']) > 0,
'num_data_pools': len(fs['data_pools']),
'standby_count_wanted': fs['standby_count_wanted'],
'approx_ctime': fs['created'][0:7],
'files': rfiles,
'bytes': rbytes,
'snaps': rsnaps,
})
num_mds += len(fs['info'])
report['fs']['total_num_mds'] = num_mds # type: ignore
# daemons
report['metadata'] = dict(osd=self.gather_osd_metadata(osd_map),
mon=self.gather_mon_metadata(mon_map))
# host counts
servers = self.list_servers()
self.log.debug('servers %s' % servers)
hosts = {
'num': len([h for h in servers if h['hostname']]),
}
for t in ['mon', 'mds', 'osd', 'mgr']:
nr_services = sum(1 for host in servers if any(
service
for service in cast(List[ServiceInfoT], host['services'])
if service['type'] == t))
hosts['num_with_' + t] = nr_services
report['hosts'] = hosts
report['usage'] = {
'pools': len(df['pools']),
'pg_num': num_pg,
'total_used_bytes': df['stats']['total_used_bytes'],
'total_bytes': df['stats']['total_bytes'],
'total_avail_bytes': df['stats']['total_avail_bytes']
}
services: DefaultDict[str, int] = defaultdict(int)
for key, value in service_map['services'].items():
services[key] += 1
if key == 'rgw':
rgw = {}
zones = set()
zonegroups = set()
frontends = set()
count = 0
d = value.get('daemons', dict())
for k, v in d.items():
if k == 'summary' and v:
rgw[k] = v
elif isinstance(v, dict) and 'metadata' in v:
count += 1
zones.add(v['metadata']['zone_id'])
zonegroups.add(v['metadata']['zonegroup_id'])
frontends.add(v['metadata']['frontend_type#0'])
# we could actually iterate over all the keys of
# the dict and check for how many frontends there
# are, but it is unlikely that one would be running
# more than 2 supported ones
f2 = v['metadata'].get('frontend_type#1', None)
if f2:
frontends.add(f2)
rgw['count'] = count
rgw['zones'] = len(zones)
rgw['zonegroups'] = len(zonegroups)
rgw['frontends'] = list(
frontends) # sets aren't json-serializable
report['rgw'] = rgw
report['services'] = services
try:
report['balancer'] = self.remote('balancer',
'gather_telemetry')
except ImportError:
report['balancer'] = {'active': False}
if 'crash' in channels:
report['crashes'] = self.gather_crashinfo()
# NOTE: We do not include the 'device' channel in this report; it is
# sent to a different endpoint.
return report
def _try_post(self, what: str, url: str,
report: Dict[str, Dict[str, str]]) -> Optional[str]:
self.log.info('Sending %s to: %s' % (what, url))
proxies = dict()
if self.proxy:
self.log.info('Send using HTTP(S) proxy: %s', self.proxy)
proxies['http'] = self.proxy
proxies['https'] = self.proxy
try:
resp = requests.put(url=url, json=report, proxies=proxies)
resp.raise_for_status()
except Exception as e:
fail_reason = 'Failed to send %s to %s: %s' % (what, url, str(e))
self.log.error(fail_reason)
return fail_reason
return None
class EndPoint(enum.Enum):
ceph = 'ceph'
device = 'device'
def send(
self,
report: Dict[str, Dict[str, str]],
endpoint: Optional[List[EndPoint]] = None) -> Tuple[int, str, str]:
if not endpoint:
endpoint = [self.EndPoint.ceph, self.EndPoint.device]
failed = []
success = []
self.log.debug('Send endpoints %s' % endpoint)
for e in endpoint:
if e == self.EndPoint.ceph:
fail_reason = self._try_post('ceph report', self.url, report)
if fail_reason:
failed.append(fail_reason)
else:
now = int(time.time())
self.last_upload = now
self.set_store('last_upload', str(now))
success.append('Ceph report sent to {0}'.format(self.url))
self.log.info('Sent report to {0}'.format(self.url))
elif e == self.EndPoint.device:
if 'device' in self.get_active_channels():
devices = self.gather_device_report()
assert devices
num_devs = 0
num_hosts = 0
for host, ls in devices.items():
self.log.debug('host %s devices %s' % (host, ls))
if not len(ls):
continue
fail_reason = self._try_post('devices',
self.device_url, ls)
if fail_reason:
failed.append(fail_reason)
else:
num_devs += len(ls)
num_hosts += 1
if num_devs:
success.append('Reported %d devices across %d hosts' %
(num_devs, len(devices)))
if failed:
return 1, '', '\n'.join(success + failed)
return 0, '', '\n'.join(success)
@CLIReadCommand('telemetry status')
def status(self) -> Tuple[int, str, str]:
'''
Show current configuration
'''
r = {}
for opt in self.MODULE_OPTIONS:
r[opt['name']] = getattr(self, opt['name'])
r['last_upload'] = (time.ctime(self.last_upload)
if self.last_upload else self.last_upload)
return 0, json.dumps(r, indent=4, sort_keys=True), ''
@CLICommand('telemetry on')
def on(self, license: Optional[str] = None) -> Tuple[int, str, str]:
'''
Enable telemetry reports from this cluster
'''
if license != LICENSE:
return -errno.EPERM, '', f'''Telemetry data is licensed under the {LICENSE_NAME} ({LICENSE_URL}).
To enable, add '--license {LICENSE}' to the 'ceph telemetry on' command.'''
else:
self.set_module_option('enabled', True)
self.set_module_option('last_opt_revision', REVISION)
return 0, '', ''
@CLICommand('telemetry off')
def off(self) -> Tuple[int, str, str]:
'''
Disable telemetry reports from this cluster
'''
self.set_module_option('enabled', False)
self.set_module_option('last_opt_revision', 1)
return 0, '', ''
@CLICommand('telemetry send')
def do_send(self,
endpoint: Optional[List[EndPoint]] = None,
license: Optional[str] = None) -> Tuple[int, str, str]:
if self.last_opt_revision < LAST_REVISION_RE_OPT_IN and license != LICENSE:
self.log.debug(('A telemetry send attempt while opted-out. '
'Asking for license agreement'))
return -errno.EPERM, '', f'''Telemetry data is licensed under the {LICENSE_NAME} ({LICENSE_URL}).
To manually send telemetry data, add '--license {LICENSE}' to the 'ceph telemetry send' command.
Please consider enabling the telemetry module with 'ceph telemetry on'.'''
else:
self.last_report = self.compile_report()
return self.send(self.last_report, endpoint)
@CLIReadCommand('telemetry show')
def show(self,
channels: Optional[List[str]] = None) -> Tuple[int, str, str]:
'''
Show report of all channels
'''
report = self.get_report(channels=channels)
report = json.dumps(report, indent=4, sort_keys=True)
if self.channel_device:
report += '''
Device report is generated separately. To see it run 'ceph telemetry show-device'.'''
return 0, report, ''
@CLIReadCommand('telemetry show-device')
def show_device(self) -> Tuple[int, str, str]:
return 0, json.dumps(self.get_report('device'),
indent=4,
sort_keys=True), ''
@CLIReadCommand('telemetry show-all')
def show_all(self) -> Tuple[int, str, str]:
return 0, json.dumps(self.get_report('all'), indent=4,
sort_keys=True), ''
def get_report(self,
report_type: str = 'default',
channels: Optional[List[str]] = None) -> Dict[str, Any]:
if report_type == 'default':
return self.compile_report(channels=channels)
elif report_type == 'device':
return self.gather_device_report()
elif report_type == 'all':
return {
'report': self.compile_report(channels=channels),
'device_report': self.gather_device_report()
}
return {}
def self_test(self) -> None:
report = self.compile_report()
if len(report) == 0:
raise RuntimeError('Report is empty')
if 'report_id' not in report:
raise RuntimeError('report_id not found in report')
def shutdown(self) -> None:
self.run = False
self.event.set()
def refresh_health_checks(self) -> None:
health_checks = {}
if self.enabled and self.last_opt_revision < LAST_REVISION_RE_OPT_IN:
health_checks['TELEMETRY_CHANGED'] = {
'severity':
'warning',
'summary':
'Telemetry requires re-opt-in',
'detail': [
'telemetry report includes new information; must re-opt-in (or out)'
]
}
self.set_health_checks(health_checks)
def serve(self) -> None:
self.load()
self.config_notify()
self.run = True
self.log.debug('Waiting for mgr to warm up')
self.event.wait(10)
while self.run:
self.event.clear()
self.refresh_health_checks()
if self.last_opt_revision < LAST_REVISION_RE_OPT_IN:
self.log.debug('Not sending report until user re-opts-in')
self.event.wait(1800)
continue
if not self.enabled:
self.log.debug('Not sending report until configured to do so')
self.event.wait(1800)
continue
now = int(time.time())
if not self.last_upload or \
(now - self.last_upload) > self.interval * 3600:
self.log.info('Compiling and sending report to %s', self.url)
try:
self.last_report = self.compile_report()
except Exception:
self.log.exception('Exception while compiling report:')
self.send(self.last_report)
else:
self.log.debug(
'Interval for sending new report has not expired')
sleep = 3600
self.log.debug('Sleeping for %d seconds', sleep)
self.event.wait(sleep)
@staticmethod
def can_run() -> Tuple[bool, str]:
return True, ''
class Module(MgrModule):
MODULE_OPTIONS = [
Option(name='hostname', default=None, desc='InfluxDB server hostname'),
Option(name='port',
type='int',
default=8086,
desc='InfluxDB server port'),
Option(name='database',
default='ceph',
desc=('InfluxDB database name. You will need to create this '
'database and grant write privileges to the configured '
'username or the username must have admin privileges to '
'create it.')),
Option(name='username',
default=None,
desc='username of InfluxDB server user'),
Option(name='password',
default=None,
desc='password of InfluxDB server user'),
Option(name='interval',
type='secs',
min=5,
default=30,
desc='Time between reports to InfluxDB. Default 30 seconds.'),
Option(
name='ssl',
default='false',
desc=
'Use https connection for InfluxDB server. Use "true" or "false".'
),
Option(
name='verify_ssl',
default='true',
desc='Verify https cert for InfluxDB server. Use "true" or "false".'
),
Option(
name='threads',
type='int',
min=1,
max=32,
default=5,
desc=
'How many worker threads should be spawned for sending data to InfluxDB.'
),
Option(
name='batch_size',
type='int',
default=5000,
desc=
'How big batches of data points should be when sending to InfluxDB.'
),
]
@property
def config_keys(self) -> Dict[str, OptionValue]:
return dict(
(o['name'], o.get('default', None)) for o in self.MODULE_OPTIONS)
COMMANDS = [{
"cmd": "influx config-set name=key,type=CephString "
"name=value,type=CephString",
"desc": "Set a configuration value",
"perm": "rw"
}, {
"cmd": "influx config-show",
"desc": "Show current configuration",
"perm": "r"
}, {
"cmd": "influx send",
"desc": "Force sending data to Influx",
"perm": "rw"
}]
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
self.event = Event()
self.run = True
self.config: Dict[str, OptionValue] = dict()
self.workers: List[Thread] = list()
self.queue: 'queue.Queue[Optional[List[Dict[str, str]]]]' = queue.Queue(
maxsize=100)
self.health_checks: Dict[str, Dict[str, Any]] = dict()
def get_fsid(self) -> str:
return self.get('mon_map')['fsid']
@staticmethod
def can_run() -> Tuple[bool, str]:
if InfluxDBClient is not None:
return True, ""
else:
return False, "influxdb python module not found"
@staticmethod
def get_timestamp() -> str:
return datetime.utcnow().isoformat() + 'Z'
@staticmethod
def chunk(l: Iterator[Dict[str, str]],
n: int) -> Iterator[List[Dict[str, str]]]:
try:
while True:
xs = []
for _ in range(n):
xs.append(next(l))
yield xs
except StopIteration:
yield xs
def queue_worker(self) -> None:
while True:
try:
points = self.queue.get()
if not points:
self.log.debug('Worker shutting down')
break
start = time.time()
with self.get_influx_client() as client:
client.write_points(points, time_precision='ms')
runtime = time.time() - start
self.log.debug('Writing points %d to Influx took %.3f seconds',
len(points), runtime)
except RequestException as e:
hostname = self.config['hostname']
port = self.config['port']
self.log.exception(
f"Failed to connect to Influx host {hostname}:{port}")
self.health_checks.update({
'MGR_INFLUX_SEND_FAILED': {
'severity': 'warning',
'summary': 'Failed to send data to InfluxDB server '
f'at {hostname}:{port} due to an connection error',
'detail': [str(e)]
}
})
except InfluxDBClientError as e:
self.health_checks.update({
'MGR_INFLUX_SEND_FAILED': {
'severity': 'warning',
'summary': 'Failed to send data to InfluxDB',
'detail': [str(e)]
}
})
self.log.exception('Failed to send data to InfluxDB')
except queue.Empty:
continue
except:
self.log.exception(
'Unhandled Exception while sending to Influx')
finally:
self.queue.task_done()
def get_latest(self, daemon_type: str, daemon_name: str, stat: str) -> int:
data = self.get_counter(daemon_type, daemon_name, stat)[stat]
if data:
return data[-1][1]
return 0
def get_df_stats(self, now) -> Tuple[List[Dict[str, Any]], Dict[str, str]]:
df = self.get("df")
data = []
pool_info = {}
df_types = [
'stored', 'kb_used', 'dirty', 'rd', 'rd_bytes', 'stored_raw', 'wr',
'wr_bytes', 'objects', 'max_avail', 'quota_objects', 'quota_bytes'
]
for df_type in df_types:
for pool in df['pools']:
point = {
"measurement": "ceph_pool_stats",
"tags": {
"pool_name": pool['name'],
"pool_id": pool['id'],
"type_instance": df_type,
"fsid": self.get_fsid()
},
"time": now,
"fields": {
"value": pool['stats'][df_type],
}
}
data.append(point)
pool_info.update({str(pool['id']): pool['name']})
return data, pool_info
def get_pg_summary_osd(self, pool_info: Dict[str, str],
now: str) -> Iterator[Dict[str, Any]]:
pg_sum = self.get('pg_summary')
osd_sum = pg_sum['by_osd']
for osd_id, stats in osd_sum.items():
metadata = self.get_metadata('osd', "%s" % osd_id)
if not metadata:
continue
for stat in stats:
yield {
"measurement": "ceph_pg_summary_osd",
"tags": {
"ceph_daemon": "osd." + str(osd_id),
"type_instance": stat,
"host": metadata['hostname']
},
"time": now,
"fields": {
"value": stats[stat]
}
}
def get_pg_summary_pool(self, pool_info: Dict[str, str],
now: str) -> Iterator[Dict[str, Any]]:
pool_sum = self.get('pg_summary')['by_pool']
for pool_id, stats in pool_sum.items():
try:
pool_name = pool_info[pool_id]
except KeyError:
self.log.error(
'Unable to find pool name for pool {}'.format(pool_id))
continue
for stat in stats:
yield {
"measurement": "ceph_pg_summary_pool",
"tags": {
"pool_name": pool_name,
"pool_id": pool_id,
"type_instance": stat,
},
"time": now,
"fields": {
"value": stats[stat],
}
}
def get_daemon_stats(self, now: str) -> Iterator[Dict[str, Any]]:
for daemon, counters in self.get_all_perf_counters().items():
svc_type, svc_id = daemon.split(".", 1)
metadata = self.get_metadata(svc_type, svc_id)
if metadata is not None:
hostname = metadata['hostname']
else:
hostname = 'N/A'
for path, counter_info in counters.items():
if counter_info['type'] & self.PERFCOUNTER_HISTOGRAM:
continue
value = counter_info['value']
yield {
"measurement": "ceph_daemon_stats",
"tags": {
"ceph_daemon": daemon,
"type_instance": path,
"host": hostname,
"fsid": self.get_fsid()
},
"time": now,
"fields": {
"value": value
}
}
def init_module_config(self) -> None:
self.config['hostname'] = \
self.get_module_option("hostname", default=self.config_keys['hostname'])
self.config['port'] = \
cast(int, self.get_module_option("port", default=self.config_keys['port']))
self.config['database'] = \
self.get_module_option("database", default=self.config_keys['database'])
self.config['username'] = \
self.get_module_option("username", default=self.config_keys['username'])
self.config['password'] = \
self.get_module_option("password", default=self.config_keys['password'])
self.config['interval'] = \
cast(int, self.get_module_option("interval",
default=self.config_keys['interval']))
self.config['threads'] = \
cast(int, self.get_module_option("threads",
default=self.config_keys['threads']))
self.config['batch_size'] = \
cast(int, self.get_module_option("batch_size",
default=self.config_keys['batch_size']))
ssl = cast(
str, self.get_module_option("ssl",
default=self.config_keys['ssl']))
self.config['ssl'] = ssl.lower() == 'true'
verify_ssl = \
cast(str, self.get_module_option("verify_ssl", default=self.config_keys['verify_ssl']))
self.config['verify_ssl'] = verify_ssl.lower() == 'true'
def gather_statistics(self) -> Iterator[Dict[str, str]]:
now = self.get_timestamp()
df_stats, pools = self.get_df_stats(now)
return chain(df_stats, self.get_daemon_stats(now),
self.get_pg_summary_osd(pools, now),
self.get_pg_summary_pool(pools, now))
@contextmanager
def get_influx_client(self) -> Iterator['InfluxDBClient']:
client = InfluxDBClient(self.config['hostname'], self.config['port'],
self.config['username'],
self.config['password'],
self.config['database'], self.config['ssl'],
self.config['verify_ssl'])
try:
yield client
finally:
try:
client.close()
except AttributeError:
# influxdb older than v5.0.0
pass
def send_to_influx(self) -> bool:
if not self.config['hostname']:
self.log.error(
"No Influx server configured, please set one using: "
"ceph influx config-set hostname <hostname>")
self.set_health_checks({
'MGR_INFLUX_NO_SERVER': {
'severity': 'warning',
'summary': 'No InfluxDB server configured',
'detail': ['Configuration option hostname not set']
}
})
return False
self.health_checks = dict()
self.log.debug("Sending data to Influx host: %s",
self.config['hostname'])
try:
with self.get_influx_client() as client:
databases = client.get_list_database()
if {'name': self.config['database']} not in databases:
self.log.info(
"Database '%s' not found, trying to create "
"(requires admin privs). You can also create "
"manually and grant write privs to user "
"'%s'", self.config['database'],
self.config['database'])
client.create_database(self.config['database'])
client.create_retention_policy(
name='8_weeks',
duration='8w',
replication='1',
default=True,
database=self.config['database'])
self.log.debug('Gathering statistics')
points = self.gather_statistics()
for chunk in self.chunk(points, cast(int,
self.config['batch_size'])):
self.queue.put(chunk, block=False)
self.log.debug('Queue currently contains %d items',
self.queue.qsize())
return True
except queue.Full:
self.health_checks.update({
'MGR_INFLUX_QUEUE_FULL': {
'severity':
'warning',
'summary':
'Failed to chunk to InfluxDB Queue',
'detail': [
'Queue is full. InfluxDB might be slow with '
'processing data'
]
}
})
self.log.error('Queue is full, failed to add chunk')
return False
except (RequestException, InfluxDBClientError) as e:
self.health_checks.update({
'MGR_INFLUX_DB_LIST_FAILED': {
'severity': 'warning',
'summary': 'Failed to list/create InfluxDB database',
'detail': [str(e)]
}
})
self.log.exception('Failed to list/create InfluxDB database')
return False
finally:
self.set_health_checks(self.health_checks)
def shutdown(self) -> None:
self.log.info('Stopping influx module')
self.run = False
self.event.set()
self.log.debug('Shutting down queue workers')
for _ in self.workers:
self.queue.put([])
self.queue.join()
for worker in self.workers:
worker.join()
def self_test(self) -> Optional[str]:
now = self.get_timestamp()
daemon_stats = list(self.get_daemon_stats(now))
assert len(daemon_stats)
df_stats, pools = self.get_df_stats(now)
result = {'daemon_stats': daemon_stats, 'df_stats': df_stats}
return json.dumps(result, indent=2, sort_keys=True)
@CLIReadCommand('influx config-show')
def config_show(self) -> Tuple[int, str, str]:
"""
Show current configuration
"""
return 0, json.dumps(self.config, sort_keys=True), ''
@CLIWriteCommand('influx config-set')
def config_set(self, key: str, value: str) -> Tuple[int, str, str]:
if not value:
return -errno.EINVAL, '', 'Value should not be empty'
self.log.debug('Setting configuration option %s to %s', key, value)
try:
self.set_module_option(key, value)
self.config[key] = self.get_module_option(key)
return 0, 'Configuration option {0} updated'.format(key), ''
except ValueError as e:
return -errno.EINVAL, '', str(e)
@CLICommand('influx send')
def send(self) -> Tuple[int, str, str]:
"""
Force sending data to Influx
"""
self.send_to_influx()
return 0, 'Sending data to Influx', ''
def serve(self) -> None:
if InfluxDBClient is None:
self.log.error("Cannot transmit statistics: influxdb python "
"module not found. Did you install it?")
return
self.log.info('Starting influx module')
self.init_module_config()
self.run = True
self.log.debug('Starting %d queue worker threads',
self.config['threads'])
for i in range(cast(int, self.config['threads'])):
worker = Thread(target=self.queue_worker, args=())
worker.setDaemon(True)
worker.start()
self.workers.append(worker)
while self.run:
start = time.time()
self.send_to_influx()
runtime = time.time() - start
self.log.debug('Finished sending data to Influx in %.3f seconds',
runtime)
self.log.debug("Sleeping for %d seconds", self.config['interval'])
self.event.wait(cast(float, self.config['interval']))
class Module(MgrModule, CherryPyConfig):
"""
dashboard module entrypoint
"""
COMMANDS = [
{
'cmd': 'dashboard set-jwt-token-ttl '
'name=seconds,type=CephInt',
'desc': 'Set the JWT token TTL in seconds',
'perm': 'w'
},
{
'cmd': 'dashboard get-jwt-token-ttl',
'desc': 'Get the JWT token TTL in seconds',
'perm': 'r'
},
{
"cmd": "dashboard create-self-signed-cert",
"desc": "Create self signed certificate",
"perm": "w"
},
{
"cmd": "dashboard grafana dashboards update",
"desc": "Push dashboards to Grafana",
"perm": "w",
},
]
COMMANDS.extend(options_command_list())
COMMANDS.extend(SSO_COMMANDS)
PLUGIN_MANAGER.hook.register_commands()
MODULE_OPTIONS = [
Option(name='server_addr', type='str', default=get_default_addr()),
Option(name='server_port', type='int', default=8080),
Option(name='ssl_server_port', type='int', default=8443),
Option(name='jwt_token_ttl', type='int', default=28800),
Option(name='url_prefix', type='str', default=''),
Option(name='key_file', type='str', default=''),
Option(name='crt_file', type='str', default=''),
Option(name='ssl', type='bool', default=True),
Option(name='standby_behaviour',
type='str',
default='redirect',
enum_allowed=['redirect', 'error']),
Option(name='standby_error_status_code',
type='int',
default=500,
min=400,
max=599)
]
MODULE_OPTIONS.extend(options_schema_list())
for options in PLUGIN_MANAGER.hook.get_options() or []:
MODULE_OPTIONS.extend(options)
__pool_stats = collections.defaultdict(lambda: collections.defaultdict(
lambda: collections.deque(maxlen=10))) # type: dict
def __init__(self, *args, **kwargs):
super(Module, self).__init__(*args, **kwargs)
CherryPyConfig.__init__(self)
mgr.init(self)
self._stopping = threading.Event()
self.shutdown_event = threading.Event()
self.ACCESS_CTRL_DB = None
self.SSO_DB = None
self.health_checks = {}
@classmethod
def can_run(cls):
if cherrypy is None:
return False, "Missing dependency: cherrypy"
if not os.path.exists(cls.get_frontend_path()):
return False, (
"Frontend assets not found at '{}': incomplete build?".format(
cls.get_frontend_path()))
return True, ""
@classmethod
def get_frontend_path(cls):
current_dir = os.path.dirname(os.path.abspath(__file__))
path = os.path.join(current_dir, 'frontend/dist')
if os.path.exists(path):
return path
else:
path = os.path.join(current_dir, '../../../../build',
'src/pybind/mgr/dashboard', 'frontend/dist')
return os.path.abspath(path)
def serve(self):
if 'COVERAGE_ENABLED' in os.environ:
import coverage
__cov = coverage.Coverage(config_file="{}/.coveragerc".format(
os.path.dirname(__file__)),
data_suffix=True)
__cov.start()
cherrypy.engine.subscribe('after_request', __cov.save)
cherrypy.engine.subscribe('stop', __cov.stop)
AuthManager.initialize()
load_sso_db()
uri = self.await_configuration()
if uri is None:
# We were shut down while waiting
return
# Publish the URI that others may use to access the service we're
# about to start serving
self.set_uri(uri)
mapper, parent_urls = generate_routes(self.url_prefix)
config = {}
for purl in parent_urls:
config[purl] = {'request.dispatch': mapper}
cherrypy.tree.mount(None, config=config)
PLUGIN_MANAGER.hook.setup()
cherrypy.engine.start()
NotificationQueue.start_queue()
TaskManager.init()
logger.info('Engine started.')
update_dashboards = str_to_bool(
self.get_module_option('GRAFANA_UPDATE_DASHBOARDS', 'False'))
if update_dashboards:
logger.info('Starting Grafana dashboard task')
TaskManager.run(
'grafana/dashboards/update',
{},
push_local_dashboards,
kwargs=dict(tries=10, sleep=60),
)
# wait for the shutdown event
self.shutdown_event.wait()
self.shutdown_event.clear()
NotificationQueue.stop()
cherrypy.engine.stop()
logger.info('Engine stopped')
def shutdown(self):
super(Module, self).shutdown()
CherryPyConfig.shutdown(self)
logger.info('Stopping engine...')
self.shutdown_event.set()
def _set_ssl_item(self,
item_label: str,
item_key: 'SslConfigKey' = 'crt',
mgr_id: Optional[str] = None,
inbuf: Optional[str] = None):
if inbuf is None:
return -errno.EINVAL, '', f'Please specify the {item_label} with "-i" option'
if mgr_id is not None:
self.set_store(_get_localized_key(mgr_id, item_key), inbuf)
else:
self.set_store(item_key, inbuf)
return 0, f'SSL {item_label} updated', ''
@CLIWriteCommand("dashboard set-ssl-certificate")
def set_ssl_certificate(self,
mgr_id: Optional[str] = None,
inbuf: Optional[str] = None):
return self._set_ssl_item('certificate', 'crt', mgr_id, inbuf)
@CLIWriteCommand("dashboard set-ssl-certificate-key")
def set_ssl_certificate_key(self,
mgr_id: Optional[str] = None,
inbuf: Optional[str] = None):
return self._set_ssl_item('certificate key', 'key', mgr_id, inbuf)
@CLIWriteCommand("dashboard create-self-signed-cert")
def set_mgr_created_self_signed_cert(self):
cert, pkey = create_self_signed_cert('IT', 'ceph-dashboard')
result = HandleCommandResult(*self.set_ssl_certificate(inbuf=cert))
if result.retval != 0:
return result
result = HandleCommandResult(*self.set_ssl_certificate_key(inbuf=pkey))
if result.retval != 0:
return result
return 0, 'Self-signed certificate created', ''
@CLICommand("dashboard get issue")
def get_issues_cli(self, issue_number: int):
try:
issue_number = int(issue_number)
except TypeError:
return -errno.EINVAL, '', f'Invalid issue number {issue_number}'
tracker_client = CephTrackerClient()
try:
response = tracker_client.get_issues(issue_number)
except RequestException as error:
if error.status_code == 404:
return -errno.EINVAL, '', f'Issue {issue_number} not found'
else:
return -errno.EREMOTEIO, '', f'Error: {str(error)}'
return 0, str(response), ''
@CLICommand("dashboard create issue")
def report_issues_cli(self, project: str, tracker: str, subject: str,
description: str):
'''
Create an issue in the Ceph Issue tracker
Syntax: ceph dashboard create issue <project> <bug|feature> <subject> <description>
'''
try:
feedback = Feedback(Feedback.Project[project].value,
Feedback.TrackerType[tracker].value, subject,
description)
except KeyError:
return -errno.EINVAL, '', 'Invalid arguments'
tracker_client = CephTrackerClient()
try:
response = tracker_client.create_issue(feedback)
except RequestException as error:
if error.status_code == 401:
return -errno.EINVAL, '', 'Invalid API Key'
else:
return -errno.EINVAL, '', f'Error: {str(error)}'
except Exception:
return -errno.EINVAL, '', 'Ceph Tracker API key not set'
return 0, str(response), ''
@CLIWriteCommand("dashboard set-rgw-credentials")
def set_rgw_credentials(self):
try:
configure_rgw_credentials()
except Exception as error:
return -errno.EINVAL, '', str(error)
return 0, 'RGW credentials configured', ''
def handle_command(self, inbuf, cmd):
# pylint: disable=too-many-return-statements
res = handle_option_command(cmd, inbuf)
if res[0] != -errno.ENOSYS:
return res
res = handle_sso_command(cmd)
if res[0] != -errno.ENOSYS:
return res
if cmd['prefix'] == 'dashboard set-jwt-token-ttl':
self.set_module_option('jwt_token_ttl', str(cmd['seconds']))
return 0, 'JWT token TTL updated', ''
if cmd['prefix'] == 'dashboard get-jwt-token-ttl':
ttl = self.get_module_option('jwt_token_ttl',
JwtManager.JWT_TOKEN_TTL)
return 0, str(ttl), ''
if cmd['prefix'] == 'dashboard grafana dashboards update':
push_local_dashboards()
return 0, 'Grafana dashboards updated', ''
return (-errno.EINVAL, '',
'Command not found \'{0}\''.format(cmd['prefix']))
def notify(self, notify_type, notify_id):
NotificationQueue.new_notification(notify_type, notify_id)
def get_updated_pool_stats(self):
df = self.get('df')
pool_stats = {p['id']: p['stats'] for p in df['pools']}
now = time.time()
for pool_id, stats in pool_stats.items():
for stat_name, stat_val in stats.items():
self.__pool_stats[pool_id][stat_name].append((now, stat_val))
return self.__pool_stats
def config_notify(self):
"""
This method is called whenever one of our config options is changed.
"""
PLUGIN_MANAGER.hook.config_notify()
def refresh_health_checks(self):
self.set_health_checks(self.health_checks)
class Module(MgrModule):
MODULE_OPTIONS = [
Option(
name='enable_monitoring',
default=True,
type='bool',
desc='monitor device health metrics',
runtime=True,
),
Option(
name='scrape_frequency',
default=86400,
type='secs',
desc='how frequently to scrape device health metrics',
runtime=True,
),
Option(
name='pool_name',
default='device_health_metrics',
type='str',
desc='name of pool in which to store device health metrics',
runtime=True,
),
Option(
name='retention_period',
default=(86400 * 180),
type='secs',
desc='how long to retain device health metrics',
runtime=True,
),
Option(
name='mark_out_threshold',
default=(86400 * 14 * 2),
type='secs',
desc='automatically mark OSD if it may fail before this long',
runtime=True,
),
Option(
name='warn_threshold',
default=(86400 * 14 * 6),
type='secs',
desc='raise health warning if OSD may fail before this long',
runtime=True,
),
Option(
name='self_heal',
default=True,
type='bool',
desc='preemptively heal cluster around devices that may fail',
runtime=True,
),
Option(
name='sleep_interval',
default=600,
type='secs',
desc='how frequently to wake up and check device health',
runtime=True,
),
]
def __init__(self, *args, **kwargs):
super(Module, self).__init__(*args, **kwargs)
# populate options (just until serve() runs)
for opt in self.MODULE_OPTIONS:
setattr(self, opt['name'], opt['default'])
# other
self.run = True
self.event = Event()
self.has_device_pool = False
def is_valid_daemon_name(self, who):
parts = who.split('.')
if len(parts) != 2:
return False
return parts[0] in ('osd', 'mon')
def handle_command(self, _, cmd):
self.log.error("handle_command")
@CLICommand('device query-daemon-health-metrics', perm='r')
def do_query_daemon_health_metrics(self, who: str):
'''
Get device health metrics for a given daemon
'''
if not self.is_valid_daemon_name(who):
return -errno.EINVAL, '', 'not a valid mon or osd daemon name'
(daemon_type, daemon_id) = who.split('.')
result = CommandResult('')
self.send_command(result, daemon_type, daemon_id,
json.dumps({
'prefix': 'smart',
'format': 'json',
}), '')
return result.wait()
@CLICommand('device scrape-daemon-health-metrics', perm='r')
def do_scrape_daemon_health_metrics(self, who: str):
'''
Scrape and store device health metrics for a given daemon
'''
if not self.is_valid_daemon_name(who):
return -errno.EINVAL, '', 'not a valid mon or osd daemon name'
(daemon_type, daemon_id) = who.split('.')
return self.scrape_daemon(daemon_type, daemon_id)
@CLICommand('device scrape-daemon-health-metrics', perm='r')
def do_scrape_health_metrics(self, devid: Optional[str] = None):
'''
Scrape and store device health metrics
'''
if devid is None:
return self.scrape_all()
else:
return self.scrape_device(devid)
@CLICommand('device get-health-metrics', perm='r')
def do_get_health_metrics(self, devid: str, sample: Optional[str] = None):
'''
Show stored device metrics for the device
'''
return self.show_device_metrics(devid, sample)
@CLICommand('device check-health', perm='rw')
def do_check_health(self):
'''
Check life expectancy of devices
'''
return self.check_health()
@CLICommand('device monitoring on', perm='rw')
def do_monitoring_on(self):
'''
Enable device health monitoring
'''
self.set_module_option('enable_monitoring', True)
self.event.set()
@CLICommand('device monitoring off', perm='rw')
def do_monitoring_off(self):
'''
Disable device health monitoring
'''
self.set_module_option('enable_monitoring', False)
self.set_health_checks({}) # avoid stuck health alerts
@CLICommand('device predict-life-expectancy', perm='r')
def do_predict_life_expectancy(self, devid: str):
'''
Predict life expectancy with local predictor
'''
return self.predict_lift_expectancy(devid)
def self_test(self):
self.config_notify()
osdmap = self.get('osd_map')
osd_id = osdmap['osds'][0]['osd']
osdmeta = self.get('osd_metadata')
devs = osdmeta.get(str(osd_id), {}).get('device_ids')
if devs:
devid = devs.split()[0].split('=')[1]
(r, before, err) = self.show_device_metrics(devid, '')
assert r == 0
(r, out, err) = self.scrape_device(devid)
assert r == 0
(r, after, err) = self.show_device_metrics(devid, '')
assert r == 0
assert before != after
def config_notify(self):
for opt in self.MODULE_OPTIONS:
setattr(self, opt['name'], self.get_module_option(opt['name']))
self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name']))
def notify(self, notify_type, notify_id):
# create device_health_metrics pool if it doesn't exist
if notify_type == "osd_map" and self.enable_monitoring:
if not self.has_device_pool:
self.create_device_pool()
self.has_device_pool = True
def create_device_pool(self):
self.log.debug('create %s pool' % self.pool_name)
# create pool
result = CommandResult('')
self.send_command(
result, 'mon', '',
json.dumps({
'prefix': 'osd pool create',
'format': 'json',
'pool': self.pool_name,
'pg_num': 1,
'pg_num_min': 1,
}), '')
r, outb, outs = result.wait()
assert r == 0
# set pool application
result = CommandResult('')
self.send_command(
result, 'mon', '',
json.dumps({
'prefix': 'osd pool application enable',
'format': 'json',
'pool': self.pool_name,
'app': 'mgr_devicehealth',
}), '')
r, outb, outs = result.wait()
assert r == 0
def serve(self):
self.log.info("Starting")
self.config_notify()
last_scrape = None
ls = self.get_store('last_scrape')
if ls:
try:
last_scrape = datetime.strptime(ls, TIME_FORMAT)
except ValueError:
pass
self.log.debug('Last scrape %s', last_scrape)
while self.run:
if self.enable_monitoring:
self.log.debug('Running')
self.check_health()
now = datetime.utcnow()
if not last_scrape:
next_scrape = now
else:
# align to scrape interval
scrape_frequency = self.scrape_frequency or 86400
seconds = (last_scrape -
datetime.utcfromtimestamp(0)).total_seconds()
seconds -= seconds % scrape_frequency
seconds += scrape_frequency
next_scrape = datetime.utcfromtimestamp(seconds)
if last_scrape:
self.log.debug('Last scrape %s, next scrape due %s',
last_scrape.strftime(TIME_FORMAT),
next_scrape.strftime(TIME_FORMAT))
else:
self.log.debug('Last scrape never, next scrape due %s',
next_scrape.strftime(TIME_FORMAT))
if now >= next_scrape:
self.scrape_all()
self.predict_all_devices()
last_scrape = now
self.set_store('last_scrape',
last_scrape.strftime(TIME_FORMAT))
# sleep
sleep_interval = self.sleep_interval or 60
self.log.debug('Sleeping for %d seconds', sleep_interval)
ret = self.event.wait(sleep_interval)
self.event.clear()
def shutdown(self):
self.log.info('Stopping')
self.run = False
self.event.set()
def open_connection(self, create_if_missing=True):
osdmap = self.get("osd_map")
assert osdmap is not None
if len(osdmap['osds']) == 0:
return None
if not self.has_device_pool:
if not create_if_missing:
return None
if self.enable_monitoring:
self.create_device_pool()
self.has_device_pool = True
ioctx = self.rados.open_ioctx(self.pool_name)
return ioctx
def scrape_daemon(self, daemon_type, daemon_id):
ioctx = self.open_connection()
if not ioctx:
return 0, "", ""
raw_smart_data = self.do_scrape_daemon(daemon_type, daemon_id)
if raw_smart_data:
for device, raw_data in raw_smart_data.items():
data = self.extract_smart_features(raw_data)
if device and data:
self.put_device_metrics(ioctx, device, data)
ioctx.close()
return 0, "", ""
def scrape_all(self):
osdmap = self.get("osd_map")
assert osdmap is not None
ioctx = self.open_connection()
if not ioctx:
return 0, "", ""
did_device = {}
ids = []
for osd in osdmap['osds']:
ids.append(('osd', str(osd['osd'])))
monmap = self.get("mon_map")
for mon in monmap['mons']:
ids.append(('mon', mon['name']))
for daemon_type, daemon_id in ids:
raw_smart_data = self.do_scrape_daemon(daemon_type, daemon_id)
if not raw_smart_data:
continue
for device, raw_data in raw_smart_data.items():
if device in did_device:
self.log.debug('skipping duplicate %s' % device)
continue
did_device[device] = 1
data = self.extract_smart_features(raw_data)
if device and data:
self.put_device_metrics(ioctx, device, data)
ioctx.close()
return 0, "", ""
def scrape_device(self, devid):
r = self.get("device " + devid)
if not r or 'device' not in r.keys():
return -errno.ENOENT, '', 'device ' + devid + ' not found'
daemons = r['device'].get('daemons', [])
if not daemons:
return (-errno.EAGAIN, '',
'device ' + devid + ' not claimed by any active daemons')
(daemon_type, daemon_id) = daemons[0].split('.')
ioctx = self.open_connection()
if not ioctx:
return 0, "", ""
raw_smart_data = self.do_scrape_daemon(daemon_type,
daemon_id,
devid=devid)
if raw_smart_data:
for device, raw_data in raw_smart_data.items():
data = self.extract_smart_features(raw_data)
if device and data:
self.put_device_metrics(ioctx, device, data)
ioctx.close()
return 0, "", ""
def do_scrape_daemon(self, daemon_type, daemon_id, devid=''):
"""
:return: a dict, or None if the scrape failed.
"""
self.log.debug('do_scrape_daemon %s.%s' % (daemon_type, daemon_id))
result = CommandResult('')
self.send_command(
result, daemon_type, daemon_id,
json.dumps({
'prefix': 'smart',
'format': 'json',
'devid': devid,
}), '')
r, outb, outs = result.wait()
try:
return json.loads(outb)
except (IndexError, ValueError):
self.log.error(
"Fail to parse JSON result from daemon {0}.{1} ({2})".format(
daemon_type, daemon_id, outb))
def put_device_metrics(self, ioctx, devid, data):
assert devid
old_key = datetime.utcnow() - timedelta(seconds=self.retention_period)
prune = old_key.strftime(TIME_FORMAT)
self.log.debug('put_device_metrics device %s prune %s' %
(devid, prune))
erase = []
try:
with rados.ReadOpCtx() as op:
# FIXME
omap_iter, ret = ioctx.get_omap_keys(op, "", MAX_SAMPLES)
assert ret == 0
ioctx.operate_read_op(op, devid)
for key, _ in list(omap_iter):
if key >= prune:
break
erase.append(key)
except rados.ObjectNotFound:
# The object doesn't already exist, no problem.
pass
except rados.Error as e:
# Do not proceed with writes if something unexpected
# went wrong with the reads.
self.log.exception("Error reading OMAP: {0}".format(e))
return
key = datetime.utcnow().strftime(TIME_FORMAT)
self.log.debug('put_device_metrics device %s key %s = %s, erase %s' %
(devid, key, data, erase))
with rados.WriteOpCtx() as op:
ioctx.set_omap(op, (key, ), (str(json.dumps(data)), ))
if len(erase):
ioctx.remove_omap_keys(op, tuple(erase))
ioctx.operate_write_op(op, devid)
def _get_device_metrics(self, devid, sample=None, min_sample=None):
res = {}
ioctx = self.open_connection(create_if_missing=False)
if not ioctx:
return {}
with ioctx:
with rados.ReadOpCtx() as op:
omap_iter, ret = ioctx.get_omap_vals(op, min_sample or '',
sample or '',
MAX_SAMPLES) # fixme
assert ret == 0
try:
ioctx.operate_read_op(op, devid)
for key, value in list(omap_iter):
if sample and key != sample:
break
if min_sample and key < min_sample:
break
try:
v = json.loads(value)
except (ValueError, IndexError):
self.log.debug(
'unable to parse value for %s: "%s"' %
(key, value))
pass
res[key] = v
except rados.ObjectNotFound:
pass
except rados.Error as e:
self.log.exception(
"RADOS error reading omap: {0}".format(e))
raise
return res
def show_device_metrics(self, devid, sample):
# verify device exists
r = self.get("device " + devid)
if not r or 'device' not in r.keys():
return -errno.ENOENT, '', 'device ' + devid + ' not found'
# fetch metrics
res = self._get_device_metrics(devid, sample=sample)
return 0, json.dumps(res, indent=4, sort_keys=True), ''
def check_health(self):
self.log.info('Check health')
config = self.get('config')
min_in_ratio = float(config.get('mon_osd_min_in_ratio'))
mark_out_threshold_td = timedelta(seconds=self.mark_out_threshold)
warn_threshold_td = timedelta(seconds=self.warn_threshold)
checks = {}
health_warnings = {
DEVICE_HEALTH: [],
DEVICE_HEALTH_IN_USE: [],
}
devs = self.get("devices")
osds_in = {}
osds_out = {}
now = datetime.utcnow()
osdmap = self.get("osd_map")
assert osdmap is not None
for dev in devs['devices']:
devid = dev['devid']
if 'life_expectancy_max' not in dev:
continue
# ignore devices that are not consumed by any daemons
if not dev['daemons']:
continue
if not dev['life_expectancy_max'] or \
dev['life_expectancy_max'] == '0.000000':
continue
# life_expectancy_(min/max) is in the format of:
# '%Y-%m-%dT%H:%M:%S.%f%z', e.g.:
# '2019-01-20T21:12:12.000000Z'
life_expectancy_max = datetime.strptime(dev['life_expectancy_max'],
'%Y-%m-%dT%H:%M:%S.%f%z')
self.log.debug('device %s expectancy max %s', dev,
life_expectancy_max)
if life_expectancy_max - now <= mark_out_threshold_td:
if self.self_heal:
# dev['daemons'] == ["osd.0","osd.1","osd.2"]
if dev['daemons']:
osds = [
x for x in dev['daemons'] if x.startswith('osd.')
]
osd_ids = map(lambda x: x[4:], osds)
for _id in osd_ids:
if self.is_osd_in(osdmap, _id):
osds_in[_id] = life_expectancy_max
else:
osds_out[_id] = 1
if life_expectancy_max - now <= warn_threshold_td:
# device can appear in more than one location in case
# of SCSI multipath
device_locations = map(lambda x: x['host'] + ':' + x['dev'],
dev['location'])
health_warnings[DEVICE_HEALTH].append(
'%s (%s); daemons %s; life expectancy between %s and %s' %
(dev['devid'], ','.join(device_locations), ','.join(
dev.get('daemons',
['none'])), dev['life_expectancy_max'],
dev.get('life_expectancy_max', 'unknown')))
# OSD might be marked 'out' (which means it has no
# data), however PGs are still attached to it.
for _id in osds_out:
num_pgs = self.get_osd_num_pgs(_id)
if num_pgs > 0:
health_warnings[DEVICE_HEALTH_IN_USE].append(
'osd.%s is marked out '
'but still has %s PG(s)' % (_id, num_pgs))
if osds_in:
self.log.debug('osds_in %s' % osds_in)
# calculate target in ratio
num_osds = len(osdmap['osds'])
num_in = len([x for x in osdmap['osds'] if x['in']])
num_bad = len(osds_in)
# sort with next-to-fail first
bad_osds = sorted(osds_in.items(), key=operator.itemgetter(1))
did = 0
to_mark_out = []
for osd_id, when in bad_osds:
ratio = float(num_in - did - 1) / float(num_osds)
if ratio < min_in_ratio:
final_ratio = float(num_in - num_bad) / float(num_osds)
checks[DEVICE_HEALTH_TOOMANY] = {
'severity':
'warning',
'summary':
HEALTH_MESSAGES[DEVICE_HEALTH_TOOMANY],
'detail': [
'%d OSDs with failing device(s) would bring "in" ratio to %f < mon_osd_min_in_ratio %f'
% (num_bad - did, final_ratio, min_in_ratio)
]
}
break
to_mark_out.append(osd_id)
did += 1
if to_mark_out:
self.mark_out_etc(to_mark_out)
for warning, ls in health_warnings.items():
n = len(ls)
if n:
checks[warning] = {
'severity': 'warning',
'summary': HEALTH_MESSAGES[warning] % n,
'count': len(ls),
'detail': ls,
}
self.set_health_checks(checks)
return 0, "", ""
def is_osd_in(self, osdmap, osd_id):
for osd in osdmap['osds']:
if str(osd_id) == str(osd['osd']):
return bool(osd['in'])
return False
def get_osd_num_pgs(self, osd_id):
stats = self.get('osd_stats')
assert stats is not None
for stat in stats['osd_stats']:
if str(osd_id) == str(stat['osd']):
return stat['num_pgs']
return -1
def mark_out_etc(self, osd_ids):
self.log.info('Marking out OSDs: %s' % osd_ids)
result = CommandResult('')
self.send_command(
result, 'mon', '',
json.dumps({
'prefix': 'osd out',
'format': 'json',
'ids': osd_ids,
}), '')
r, outb, outs = result.wait()
if r != 0:
self.log.warning(
'Could not mark OSD %s out. r: [%s], outb: [%s], outs: [%s]',
osd_ids, r, outb, outs)
for osd_id in osd_ids:
result = CommandResult('')
self.send_command(
result, 'mon', '',
json.dumps({
'prefix': 'osd primary-affinity',
'format': 'json',
'id': int(osd_id),
'weight': 0.0,
}), '')
r, outb, outs = result.wait()
if r != 0:
self.log.warning(
'Could not set osd.%s primary-affinity, '
'r: [%s], outb: [%s], outs: [%s]', osd_id, r, outb, outs)
def extract_smart_features(self, raw):
# FIXME: extract and normalize raw smartctl --json output and
# generate a dict of the fields we care about.
return raw
def predict_lift_expectancy(self, devid):
plugin_name = ''
model = self.get_ceph_option('device_failure_prediction_mode')
if model and model.lower() == 'local':
plugin_name = 'diskprediction_local'
else:
return -1, '', 'unable to enable any disk prediction model[local/cloud]'
try:
can_run, _ = self.remote(plugin_name, 'can_run')
if can_run:
return self.remote(plugin_name,
'predict_life_expectancy',
devid=devid)
except:
return -1, '', 'unable to invoke diskprediction local or remote plugin'
def predict_all_devices(self):
plugin_name = ''
model = self.get_ceph_option('device_failure_prediction_mode')
if model and model.lower() == 'local':
plugin_name = 'diskprediction_local'
else:
return -1, '', 'unable to enable any disk prediction model[local/cloud]'
try:
can_run, _ = self.remote(plugin_name, 'can_run')
if can_run:
return self.remote(plugin_name, 'predict_all_devices')
except:
return -1, '', 'unable to invoke diskprediction local or remote plugin'
def get_recent_device_metrics(self, devid, min_sample):
return self._get_device_metrics(devid, min_sample=min_sample)
def get_time_format(self):
return TIME_FORMAT
class Module(MgrModule):
MODULE_OPTIONS = [
Option(name='sleep_interval',
default=600),
Option(name='predict_interval',
default=86400),
Option(name='predictor_model',
default='prophetstor')
]
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
# options
for opt in self.MODULE_OPTIONS:
setattr(self, opt['name'], opt['default'])
# other
self._run = True
self._event = Event()
# for mypy which does not run the code
if TYPE_CHECKING:
self.sleep_interval = 0
self.predict_interval = 0
self.predictor_model = ''
def config_notify(self) -> None:
for opt in self.MODULE_OPTIONS:
setattr(self,
opt['name'],
self.get_module_option(opt['name']))
self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name']))
if self.get_ceph_option('device_failure_prediction_mode') == 'local':
self._event.set()
def refresh_config(self) -> None:
for opt in self.MODULE_OPTIONS:
setattr(self,
opt['name'],
self.get_module_option(opt['name']))
self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name']))
def self_test(self) -> None:
self.log.debug('self_test enter')
ret, out, err = self.predict_all_devices()
assert ret == 0
def serve(self) -> None:
self.log.info('Starting diskprediction local module')
self.config_notify()
last_predicted = None
ls = self.get_store('last_predicted')
if ls:
try:
last_predicted = datetime.datetime.strptime(ls, TIME_FORMAT)
except ValueError:
pass
self.log.debug('Last predicted %s', last_predicted)
while self._run:
self.refresh_config()
mode = self.get_ceph_option('device_failure_prediction_mode')
if mode == 'local':
now = datetime.datetime.utcnow()
if not last_predicted:
next_predicted = now
else:
predicted_frequency = self.predict_interval or 86400
seconds = (last_predicted - datetime.datetime.utcfromtimestamp(0)).total_seconds()
seconds -= seconds % predicted_frequency
seconds += predicted_frequency
next_predicted = datetime.datetime.utcfromtimestamp(seconds)
self.log.debug('Last scrape %s, next scrape due %s',
last_predicted.strftime(TIME_FORMAT),
next_predicted.strftime(TIME_FORMAT))
if now >= next_predicted:
self.predict_all_devices()
last_predicted = now
self.set_store('last_predicted', last_predicted.strftime(TIME_FORMAT))
sleep_interval = self.sleep_interval or 60
self.log.debug('Sleeping for %d seconds', sleep_interval)
self._event.wait(sleep_interval)
self._event.clear()
def shutdown(self) -> None:
self.log.info('Stopping')
self._run = False
self._event.set()
@staticmethod
def _convert_timestamp(predicted_timestamp: int, life_expectancy_day: int) -> str:
"""
:param predicted_timestamp: unit is nanoseconds
:param life_expectancy_day: unit is seconds
:return:
date format '%Y-%m-%d' ex. 2018-01-01
"""
return datetime.datetime.fromtimestamp(
predicted_timestamp / (1000 ** 3) + life_expectancy_day).strftime('%Y-%m-%d')
def _predict_life_expentancy(self, devid: str) -> str:
predicted_result = ''
health_data: Dict[str, Dict[str, Any]] = {}
predict_datas: List[DevSmartT] = []
try:
r, outb, outs = self.remote(
'devicehealth', 'show_device_metrics', devid=devid, sample='')
if r != 0:
self.log.error('failed to get device %s health', devid)
health_data = {}
else:
health_data = json.loads(outb)
except Exception as e:
self.log.error('failed to get device %s health data due to %s', devid, str(e))
# initialize appropriate disk failure predictor model
obj_predictor = Predictor.create(self.predictor_model)
if obj_predictor is None:
self.log.error('invalid value received for MODULE_OPTIONS.predictor_model')
return predicted_result
try:
obj_predictor.initialize(
"{}/models/{}".format(get_diskfailurepredictor_path(), self.predictor_model))
except Exception as e:
self.log.error('Error initializing predictor: %s', e)
return predicted_result
if len(health_data) >= 6:
o_keys = sorted(health_data.keys(), reverse=True)
for o_key in o_keys:
# get values for current day (?)
dev_smart = {}
s_val = health_data[o_key]
# add all smart attributes
ata_smart = s_val.get('ata_smart_attributes', {})
for attr in ata_smart.get('table', []):
# get raw smart values
if attr.get('raw', {}).get('string') is not None:
if str(attr.get('raw', {}).get('string', '0')).isdigit():
dev_smart['smart_%s_raw' % attr.get('id')] = \
int(attr.get('raw', {}).get('string', '0'))
else:
if str(attr.get('raw', {}).get('string', '0')).split(' ')[0].isdigit():
dev_smart['smart_%s_raw' % attr.get('id')] = \
int(attr.get('raw', {}).get('string',
'0').split(' ')[0])
else:
dev_smart['smart_%s_raw' % attr.get('id')] = \
attr.get('raw', {}).get('value', 0)
# get normalized smart values
if attr.get('value') is not None:
dev_smart['smart_%s_normalized' % attr.get('id')] = \
attr.get('value')
# add power on hours manually if not available in smart attributes
power_on_time = s_val.get('power_on_time', {}).get('hours')
if power_on_time is not None:
dev_smart['smart_9_raw'] = int(power_on_time)
# add device capacity
user_capacity = s_val.get('user_capacity', {}).get('bytes')
if user_capacity is not None:
dev_smart['user_capacity'] = user_capacity
else:
self.log.debug('user_capacity not found in smart attributes list')
# add device model
model_name = s_val.get('model_name')
if model_name is not None:
dev_smart['model_name'] = model_name
# add vendor
vendor = s_val.get('vendor')
if vendor is not None:
dev_smart['vendor'] = vendor
# if smart data was found, then add that to list
if dev_smart:
predict_datas.append(dev_smart)
if len(predict_datas) >= 12:
break
else:
self.log.error('unable to predict device due to health data records less than 6 days')
if len(predict_datas) >= 6:
predicted_result = obj_predictor.predict(predict_datas)
return predicted_result
def predict_life_expectancy(self, devid: str) -> Tuple[int, str, str]:
result = self._predict_life_expentancy(devid)
if result.lower() == 'good':
return 0, '>6w', ''
elif result.lower() == 'warning':
return 0, '>=2w and <=6w', ''
elif result.lower() == 'bad':
return 0, '<2w', ''
else:
return 0, 'unknown', ''
def _reset_device_life_expectancy(self, device_id: str) -> int:
result = CommandResult('')
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'device rm-life-expectancy',
'devid': device_id
}), '')
ret, _, outs = result.wait()
if ret != 0:
self.log.error(
'failed to reset device life expectancy, %s' % outs)
return ret
def _set_device_life_expectancy(self,
device_id: str,
from_date: str,
to_date: Optional[str] = None) -> int:
result = CommandResult('')
if to_date is None:
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'device set-life-expectancy',
'devid': device_id,
'from': from_date
}), '')
else:
self.send_command(result, 'mon', '', json.dumps({
'prefix': 'device set-life-expectancy',
'devid': device_id,
'from': from_date,
'to': to_date
}), '')
ret, _, outs = result.wait()
if ret != 0:
self.log.error(
'failed to set device life expectancy, %s' % outs)
return ret
def predict_all_devices(self) -> Tuple[int, str, str]:
self.log.debug('predict_all_devices')
devices = self.get('devices').get('devices', [])
for devInfo in devices:
if not devInfo.get('daemons'):
continue
if not devInfo.get('devid'):
continue
self.log.debug('%s' % devInfo)
result = self._predict_life_expentancy(devInfo['devid'])
if result == 'unknown':
self._reset_device_life_expectancy(devInfo['devid'])
continue
predicted = int(time.time() * (1000 ** 3))
if result.lower() == 'good':
life_expectancy_day_min = (TIME_WEEK * 6) + TIME_DAYS
life_expectancy_day_max = 0
elif result.lower() == 'warning':
life_expectancy_day_min = (TIME_WEEK * 2)
life_expectancy_day_max = (TIME_WEEK * 6)
elif result.lower() == 'bad':
life_expectancy_day_min = 0
life_expectancy_day_max = (TIME_WEEK * 2) - TIME_DAYS
else:
predicted = 0
life_expectancy_day_min = 0
life_expectancy_day_max = 0
if predicted and devInfo['devid'] and life_expectancy_day_min:
from_date = None
to_date = None
try:
assert life_expectancy_day_min
from_date = self._convert_timestamp(predicted, life_expectancy_day_min)
if life_expectancy_day_max:
to_date = self._convert_timestamp(predicted, life_expectancy_day_max)
self._set_device_life_expectancy(devInfo['devid'], from_date, to_date)
self._logger.info(
'succeed to set device {} life expectancy from: {}, to: {}'.format(
devInfo['devid'], from_date, to_date))
except Exception as e:
self._logger.error(
'failed to set device {} life expectancy from: {}, to: {}, {}'.format(
devInfo['devid'], from_date, to_date, str(e)))
else:
self._reset_device_life_expectancy(devInfo['devid'])
return 0, 'succeed to predicted all devices', ''
class Module(orchestrator.OrchestratorClientMixin, MgrModule):
COMMANDS = [
{
'cmd': 'fs volume ls',
'desc': "List volumes",
'perm': 'r'
},
{
'cmd':
'fs volume create '
f'name=name,type=CephString,goodchars={goodchars} '
'name=placement,type=CephString,req=false ',
'desc':
"Create a CephFS volume",
'perm':
'rw'
},
{
'cmd': 'fs volume rm '
'name=vol_name,type=CephString '
'name=yes-i-really-mean-it,type=CephString,req=false ',
'desc':
"Delete a FS volume by passing --yes-i-really-mean-it flag",
'perm': 'rw'
},
{
'cmd': 'fs subvolumegroup ls '
'name=vol_name,type=CephString ',
'desc': "List subvolumegroups",
'perm': 'r'
},
{
'cmd':
'fs subvolumegroup create '
'name=vol_name,type=CephString '
f'name=group_name,type=CephString,goodchars={goodchars} '
'name=pool_layout,type=CephString,req=false '
'name=uid,type=CephInt,req=false '
'name=gid,type=CephInt,req=false '
'name=mode,type=CephString,req=false ',
'desc':
"Create a CephFS subvolume group in a volume, and optionally, "
"with a specific data pool layout, and a specific numeric mode",
'perm':
'rw'
},
{
'cmd':
'fs subvolumegroup rm '
'name=vol_name,type=CephString '
'name=group_name,type=CephString '
'name=force,type=CephBool,req=false ',
'desc':
"Delete a CephFS subvolume group in a volume",
'perm':
'rw'
},
{
'cmd': 'fs subvolume ls '
'name=vol_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc': "List subvolumes",
'perm': 'r'
},
{
'cmd':
'fs subvolume create '
'name=vol_name,type=CephString '
f'name=sub_name,type=CephString,goodchars={goodchars} '
'name=size,type=CephInt,req=false '
'name=group_name,type=CephString,req=false '
'name=pool_layout,type=CephString,req=false '
'name=uid,type=CephInt,req=false '
'name=gid,type=CephInt,req=false '
'name=mode,type=CephString,req=false '
'name=namespace_isolated,type=CephBool,req=false ',
'desc':
"Create a CephFS subvolume in a volume, and optionally, "
"with a specific size (in bytes), a specific data pool layout, "
"a specific mode, in a specific subvolume group and in separate "
"RADOS namespace",
'perm':
'rw'
},
{
'cmd':
'fs subvolume rm '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=group_name,type=CephString,req=false '
'name=force,type=CephBool,req=false '
'name=retain_snapshots,type=CephBool,req=false ',
'desc':
"Delete a CephFS subvolume in a volume, and optionally, "
"in a specific subvolume group, force deleting a cancelled or failed "
"clone, and retaining existing subvolume snapshots",
'perm':
'rw'
},
{
'cmd':
'fs subvolume authorize '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=auth_id,type=CephString '
'name=group_name,type=CephString,req=false '
'name=access_level,type=CephString,req=false '
'name=tenant_id,type=CephString,req=false '
'name=allow_existing_id,type=CephBool,req=false ',
'desc':
"Allow a cephx auth ID access to a subvolume",
'perm':
'rw'
},
{
'cmd':
'fs subvolume deauthorize '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=auth_id,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"Deny a cephx auth ID access to a subvolume",
'perm':
'rw'
},
{
'cmd':
'fs subvolume authorized_list '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"List auth IDs that have access to a subvolume",
'perm':
'r'
},
{
'cmd':
'fs subvolume evict '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=auth_id,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"Evict clients based on auth IDs and subvolume mounted",
'perm':
'rw'
},
{
'cmd': 'fs subvolumegroup getpath '
'name=vol_name,type=CephString '
'name=group_name,type=CephString ',
'desc':
"Get the mountpath of a CephFS subvolume group in a volume",
'perm': 'r'
},
{
'cmd':
'fs subvolume getpath '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"Get the mountpath of a CephFS subvolume in a volume, "
"and optionally, in a specific subvolume group",
'perm':
'rw'
},
{
'cmd':
'fs subvolume info '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"Get the metadata of a CephFS subvolume in a volume, "
"and optionally, in a specific subvolume group",
'perm':
'r'
},
{
'cmd':
'fs subvolumegroup pin'
' name=vol_name,type=CephString'
' name=group_name,type=CephString,req=true'
' name=pin_type,type=CephChoices,strings=export|distributed|random'
' name=pin_setting,type=CephString,req=true',
'desc':
"Set MDS pinning policy for subvolumegroup",
'perm':
'rw'
},
{
'cmd':
'fs subvolumegroup snapshot ls '
'name=vol_name,type=CephString '
'name=group_name,type=CephString ',
'desc':
"List subvolumegroup snapshots",
'perm':
'r'
},
{
'cmd':
'fs subvolumegroup snapshot create '
'name=vol_name,type=CephString '
'name=group_name,type=CephString '
'name=snap_name,type=CephString ',
'desc':
"Create a snapshot of a CephFS subvolume group in a volume",
'perm':
'rw'
},
{
'cmd':
'fs subvolumegroup snapshot rm '
'name=vol_name,type=CephString '
'name=group_name,type=CephString '
'name=snap_name,type=CephString '
'name=force,type=CephBool,req=false ',
'desc':
"Delete a snapshot of a CephFS subvolume group in a volume",
'perm':
'rw'
},
{
'cmd':
'fs subvolume snapshot ls '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"List subvolume snapshots",
'perm':
'r'
},
{
'cmd':
'fs subvolume snapshot create '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=snap_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"Create a snapshot of a CephFS subvolume in a volume, "
"and optionally, in a specific subvolume group",
'perm':
'rw'
},
{
'cmd':
'fs subvolume snapshot info '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=snap_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"Get the metadata of a CephFS subvolume snapshot "
"and optionally, in a specific subvolume group",
'perm':
'r'
},
{
'cmd':
'fs subvolume snapshot rm '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=snap_name,type=CephString '
'name=group_name,type=CephString,req=false '
'name=force,type=CephBool,req=false ',
'desc':
"Delete a snapshot of a CephFS subvolume in a volume, "
"and optionally, in a specific subvolume group",
'perm':
'rw'
},
{
'cmd':
'fs subvolume resize '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=new_size,type=CephString,req=true '
'name=group_name,type=CephString,req=false '
'name=no_shrink,type=CephBool,req=false ',
'desc':
"Resize a CephFS subvolume",
'perm':
'rw'
},
{
'cmd':
'fs subvolume pin'
' name=vol_name,type=CephString'
' name=sub_name,type=CephString'
' name=pin_type,type=CephChoices,strings=export|distributed|random'
' name=pin_setting,type=CephString,req=true'
' name=group_name,type=CephString,req=false',
'desc':
"Set MDS pinning policy for subvolume",
'perm':
'rw'
},
{
'cmd':
'fs subvolume snapshot protect '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=snap_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"(deprecated) Protect snapshot of a CephFS subvolume in a volume, "
"and optionally, in a specific subvolume group",
'perm':
'rw'
},
{
'cmd':
'fs subvolume snapshot unprotect '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=snap_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"(deprecated) Unprotect a snapshot of a CephFS subvolume in a volume, "
"and optionally, in a specific subvolume group",
'perm':
'rw'
},
{
'cmd':
'fs subvolume snapshot clone '
'name=vol_name,type=CephString '
'name=sub_name,type=CephString '
'name=snap_name,type=CephString '
'name=target_sub_name,type=CephString '
'name=pool_layout,type=CephString,req=false '
'name=group_name,type=CephString,req=false '
'name=target_group_name,type=CephString,req=false ',
'desc':
"Clone a snapshot to target subvolume",
'perm':
'rw'
},
{
'cmd':
'fs clone status '
'name=vol_name,type=CephString '
'name=clone_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"Get status on a cloned subvolume.",
'perm':
'r'
},
{
'cmd':
'fs clone cancel '
'name=vol_name,type=CephString '
'name=clone_name,type=CephString '
'name=group_name,type=CephString,req=false ',
'desc':
"Cancel an pending or ongoing clone operation.",
'perm':
'r'
},
{
'cmd':
'nfs export create cephfs '
'name=fsname,type=CephString '
'name=clusterid,type=CephString '
'name=binding,type=CephString '
'name=readonly,type=CephBool,req=false '
'name=path,type=CephString,req=false ',
'desc':
"Create a cephfs export",
'perm':
'rw'
},
{
'cmd': 'nfs export delete '
'name=clusterid,type=CephString '
'name=binding,type=CephString ',
'desc': "Delete a cephfs export",
'perm': 'rw'
},
{
'cmd': 'nfs export ls '
'name=clusterid,type=CephString '
'name=detailed,type=CephBool,req=false ',
'desc': "List exports of a NFS cluster",
'perm': 'r'
},
{
'cmd': 'nfs export get '
'name=clusterid,type=CephString '
'name=binding,type=CephString ',
'desc':
"Fetch a export of a NFS cluster given the pseudo path/binding",
'perm': 'r'
},
{
'cmd':
'nfs cluster create '
'name=type,type=CephString '
f'name=clusterid,type=CephString,goodchars={goodchars} '
'name=placement,type=CephString,req=false ',
'desc':
"Create an NFS Cluster",
'perm':
'rw'
},
{
'cmd': 'nfs cluster update '
'name=clusterid,type=CephString '
'name=placement,type=CephString ',
'desc': "Updates an NFS Cluster",
'perm': 'rw'
},
{
'cmd': 'nfs cluster delete '
'name=clusterid,type=CephString ',
'desc': "Deletes an NFS Cluster",
'perm': 'rw'
},
{
'cmd': 'nfs cluster ls ',
'desc': "List NFS Clusters",
'perm': 'r'
},
{
'cmd': 'nfs cluster info '
'name=clusterid,type=CephString,req=false ',
'desc': "Displays NFS Cluster info",
'perm': 'r'
},
{
'cmd': 'nfs cluster config set '
'name=clusterid,type=CephString ',
'desc': "Set NFS-Ganesha config by `-i <config_file>`",
'perm': 'rw'
},
{
'cmd': 'nfs cluster config reset '
'name=clusterid,type=CephString ',
'desc': "Reset NFS-Ganesha Config to default",
'perm': 'rw'
},
# volume ls [recursive]
# subvolume ls <volume>
# volume authorize/deauthorize
# subvolume authorize/deauthorize
# volume describe (free space, etc)
# volume auth list (vc.get_authorized_ids)
# snapshots?
# FIXME: we're doing CephFSVolumeClient.recover on every
# path where we instantiate and connect a client. Perhaps
# keep clients alive longer, or just pass a "don't recover"
# flag in if it's the >1st time we connected a particular
# volume in the lifetime of this module instance.
]
MODULE_OPTIONS = [
Option(
'max_concurrent_clones',
type='int',
default=4,
desc='Number of asynchronous cloner threads',
)
]
def __init__(self, *args, **kwargs):
self.inited = False
# for mypy
self.max_concurrent_clones = None
self.lock = threading.Lock()
super(Module, self).__init__(*args, **kwargs)
# Initialize config option members
self.config_notify()
with self.lock:
self.vc = VolumeClient(self)
self.fs_export = FSExport(self)
self.nfs = NFSCluster(self)
self.inited = True
def __del__(self):
self.vc.shutdown()
def shutdown(self):
self.vc.shutdown()
def config_notify(self):
"""
This method is called whenever one of our config options is changed.
"""
with self.lock:
for opt in self.MODULE_OPTIONS:
setattr(
self,
opt['name'], # type: ignore
self.get_module_option(opt['name'])) # type: ignore
self.log.debug(' mgr option %s = %s', opt['name'],
getattr(self, opt['name'])) # type: ignore
if self.inited:
if opt['name'] == "max_concurrent_clones":
self.vc.cloner.reconfigure_max_concurrent_clones(
self.max_concurrent_clones)
def handle_command(self, inbuf, cmd):
handler_name = "_cmd_" + cmd['prefix'].replace(" ", "_")
try:
handler = getattr(self, handler_name)
except AttributeError:
return -errno.EINVAL, "", "Unknown command"
return handler(inbuf, cmd)
@mgr_cmd_wrap
def _cmd_fs_volume_create(self, inbuf, cmd):
vol_id = cmd['name']
placement = cmd.get('placement', '')
return self.vc.create_fs_volume(vol_id, placement)
@mgr_cmd_wrap
def _cmd_fs_volume_rm(self, inbuf, cmd):
vol_name = cmd['vol_name']
confirm = cmd.get('yes-i-really-mean-it', None)
return self.vc.delete_fs_volume(vol_name, confirm)
@mgr_cmd_wrap
def _cmd_fs_volume_ls(self, inbuf, cmd):
return self.vc.list_fs_volumes()
@mgr_cmd_wrap
def _cmd_fs_subvolumegroup_create(self, inbuf, cmd):
"""
:return: a 3-tuple of return code(int), empty string(str), error message (str)
"""
return self.vc.create_subvolume_group(vol_name=cmd['vol_name'],
group_name=cmd['group_name'],
pool_layout=cmd.get(
'pool_layout', None),
mode=cmd.get('mode', '755'),
uid=cmd.get('uid', None),
gid=cmd.get('gid', None))
@mgr_cmd_wrap
def _cmd_fs_subvolumegroup_rm(self, inbuf, cmd):
"""
:return: a 3-tuple of return code(int), empty string(str), error message (str)
"""
return self.vc.remove_subvolume_group(vol_name=cmd['vol_name'],
group_name=cmd['group_name'],
force=cmd.get('force', False))
@mgr_cmd_wrap
def _cmd_fs_subvolumegroup_ls(self, inbuf, cmd):
return self.vc.list_subvolume_groups(vol_name=cmd['vol_name'])
@mgr_cmd_wrap
def _cmd_fs_subvolume_create(self, inbuf, cmd):
"""
:return: a 3-tuple of return code(int), empty string(str), error message (str)
"""
return self.vc.create_subvolume(
vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
group_name=cmd.get('group_name', None),
size=cmd.get('size', None),
pool_layout=cmd.get('pool_layout', None),
uid=cmd.get('uid', None),
gid=cmd.get('gid', None),
mode=cmd.get('mode', '755'),
namespace_isolated=cmd.get('namespace_isolated', False))
@mgr_cmd_wrap
def _cmd_fs_subvolume_rm(self, inbuf, cmd):
"""
:return: a 3-tuple of return code(int), empty string(str), error message (str)
"""
return self.vc.remove_subvolume(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
group_name=cmd.get('group_name', None),
force=cmd.get('force', False),
retain_snapshots=cmd.get(
'retain_snapshots', False))
@mgr_cmd_wrap
def _cmd_fs_subvolume_authorize(self, inbuf, cmd):
"""
:return: a 3-tuple of return code(int), secret key(str), error message (str)
"""
return self.vc.authorize_subvolume(
vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
auth_id=cmd['auth_id'],
group_name=cmd.get('group_name', None),
access_level=cmd.get('access_level', 'rw'),
tenant_id=cmd.get('tenant_id', None),
allow_existing_id=cmd.get('allow_existing_id', False))
@mgr_cmd_wrap
def _cmd_fs_subvolume_deauthorize(self, inbuf, cmd):
"""
:return: a 3-tuple of return code(int), empty string(str), error message (str)
"""
return self.vc.deauthorize_subvolume(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
auth_id=cmd['auth_id'],
group_name=cmd.get(
'group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolume_authorized_list(self, inbuf, cmd):
"""
:return: a 3-tuple of return code(int), list of authids(json), error message (str)
"""
return self.vc.authorized_list(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
group_name=cmd.get('group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolume_evict(self, inbuf, cmd):
"""
:return: a 3-tuple of return code(int), empyt string(str), error message (str)
"""
return self.vc.evict(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
auth_id=cmd['auth_id'],
group_name=cmd.get('group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolume_ls(self, inbuf, cmd):
return self.vc.list_subvolumes(vol_name=cmd['vol_name'],
group_name=cmd.get('group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolumegroup_getpath(self, inbuf, cmd):
return self.vc.getpath_subvolume_group(vol_name=cmd['vol_name'],
group_name=cmd['group_name'])
@mgr_cmd_wrap
def _cmd_fs_subvolume_getpath(self, inbuf, cmd):
return self.vc.subvolume_getpath(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
group_name=cmd.get(
'group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolume_info(self, inbuf, cmd):
return self.vc.subvolume_info(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
group_name=cmd.get('group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolumegroup_pin(self, inbuf, cmd):
return self.vc.pin_subvolume_group(vol_name=cmd['vol_name'],
group_name=cmd['group_name'],
pin_type=cmd['pin_type'],
pin_setting=cmd['pin_setting'])
@mgr_cmd_wrap
def _cmd_fs_subvolumegroup_snapshot_create(self, inbuf, cmd):
return self.vc.create_subvolume_group_snapshot(
vol_name=cmd['vol_name'],
group_name=cmd['group_name'],
snap_name=cmd['snap_name'])
@mgr_cmd_wrap
def _cmd_fs_subvolumegroup_snapshot_rm(self, inbuf, cmd):
return self.vc.remove_subvolume_group_snapshot(
vol_name=cmd['vol_name'],
group_name=cmd['group_name'],
snap_name=cmd['snap_name'],
force=cmd.get('force', False))
@mgr_cmd_wrap
def _cmd_fs_subvolumegroup_snapshot_ls(self, inbuf, cmd):
return self.vc.list_subvolume_group_snapshots(
vol_name=cmd['vol_name'], group_name=cmd['group_name'])
@mgr_cmd_wrap
def _cmd_fs_subvolume_snapshot_create(self, inbuf, cmd):
return self.vc.create_subvolume_snapshot(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
snap_name=cmd['snap_name'],
group_name=cmd.get(
'group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolume_snapshot_rm(self, inbuf, cmd):
return self.vc.remove_subvolume_snapshot(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
snap_name=cmd['snap_name'],
group_name=cmd.get(
'group_name', None),
force=cmd.get('force', False))
@mgr_cmd_wrap
def _cmd_fs_subvolume_snapshot_info(self, inbuf, cmd):
return self.vc.subvolume_snapshot_info(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
snap_name=cmd['snap_name'],
group_name=cmd.get(
'group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolume_snapshot_ls(self, inbuf, cmd):
return self.vc.list_subvolume_snapshots(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
group_name=cmd.get(
'group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolume_resize(self, inbuf, cmd):
return self.vc.resize_subvolume(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
new_size=cmd['new_size'],
group_name=cmd.get('group_name', None),
no_shrink=cmd.get('no_shrink', False))
@mgr_cmd_wrap
def _cmd_fs_subvolume_pin(self, inbuf, cmd):
return self.vc.subvolume_pin(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
pin_type=cmd['pin_type'],
pin_setting=cmd['pin_setting'],
group_name=cmd.get('group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolume_snapshot_protect(self, inbuf, cmd):
return self.vc.protect_subvolume_snapshot(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
snap_name=cmd['snap_name'],
group_name=cmd.get(
'group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolume_snapshot_unprotect(self, inbuf, cmd):
return self.vc.unprotect_subvolume_snapshot(vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
snap_name=cmd['snap_name'],
group_name=cmd.get(
'group_name', None))
@mgr_cmd_wrap
def _cmd_fs_subvolume_snapshot_clone(self, inbuf, cmd):
return self.vc.clone_subvolume_snapshot(
vol_name=cmd['vol_name'],
sub_name=cmd['sub_name'],
snap_name=cmd['snap_name'],
group_name=cmd.get('group_name', None),
pool_layout=cmd.get('pool_layout', None),
target_sub_name=cmd['target_sub_name'],
target_group_name=cmd.get('target_group_name', None))
@mgr_cmd_wrap
def _cmd_fs_clone_status(self, inbuf, cmd):
return self.vc.clone_status(vol_name=cmd['vol_name'],
clone_name=cmd['clone_name'],
group_name=cmd.get('group_name', None))
@mgr_cmd_wrap
def _cmd_fs_clone_cancel(self, inbuf, cmd):
return self.vc.clone_cancel(vol_name=cmd['vol_name'],
clone_name=cmd['clone_name'],
group_name=cmd.get('group_name', None))
@mgr_cmd_wrap
def _cmd_nfs_export_create_cephfs(self, inbuf, cmd):
#TODO Extend export creation for rgw.
return self.fs_export.create_export(fs_name=cmd['fsname'],
cluster_id=cmd['clusterid'],
pseudo_path=cmd['binding'],
read_only=cmd.get(
'readonly', False),
path=cmd.get('path', '/'))
@mgr_cmd_wrap
def _cmd_nfs_export_delete(self, inbuf, cmd):
return self.fs_export.delete_export(cluster_id=cmd['clusterid'],
pseudo_path=cmd['binding'])
@mgr_cmd_wrap
def _cmd_nfs_export_ls(self, inbuf, cmd):
return self.fs_export.list_exports(cluster_id=cmd['clusterid'],
detailed=cmd.get('detailed', False))
@mgr_cmd_wrap
def _cmd_nfs_export_get(self, inbuf, cmd):
return self.fs_export.get_export(cluster_id=cmd['clusterid'],
pseudo_path=cmd['binding'])
@mgr_cmd_wrap
def _cmd_nfs_cluster_create(self, inbuf, cmd):
return self.nfs.create_nfs_cluster(cluster_id=cmd['clusterid'],
export_type=cmd['type'],
placement=cmd.get(
'placement', None))
@mgr_cmd_wrap
def _cmd_nfs_cluster_update(self, inbuf, cmd):
return self.nfs.update_nfs_cluster(cluster_id=cmd['clusterid'],
placement=cmd['placement'])
@mgr_cmd_wrap
def _cmd_nfs_cluster_delete(self, inbuf, cmd):
return self.nfs.delete_nfs_cluster(cluster_id=cmd['clusterid'])
@mgr_cmd_wrap
def _cmd_nfs_cluster_ls(self, inbuf, cmd):
return self.nfs.list_nfs_cluster()
@mgr_cmd_wrap
def _cmd_nfs_cluster_info(self, inbuf, cmd):
return self.nfs.show_nfs_cluster_info(
cluster_id=cmd.get('clusterid', None))
def _cmd_nfs_cluster_config_set(self, inbuf, cmd):
return self.nfs.set_nfs_cluster_config(cluster_id=cmd['clusterid'],
nfs_config=inbuf)
def _cmd_nfs_cluster_config_reset(self, inbuf, cmd):
return self.nfs.reset_nfs_cluster_config(cluster_id=cmd['clusterid'])
class Module(MgrModule):
metadata_keys = [
"arch",
"ceph_version",
"os",
"cpu",
"kernel_description",
"kernel_version",
"distro_description",
"distro"
]
MODULE_OPTIONS = [
Option(name='url',
type='str',
default='https://telemetry.ceph.com/report'),
Option(name='device_url',
type='str',
default='https://telemetry.ceph.com/device'),
Option(name='enabled',
type='bool',
default=False),
Option(name='last_opt_revision',
type='int',
default=1),
Option(name='leaderboard',
type='bool',
default=False),
Option(name='description',
type='str',
default=None),
Option(name='contact',
type='str',
default=None),
Option(name='organization',
type='str',
default=None),
Option(name='proxy',
type='str',
default=None),
Option(name='interval',
type='int',
default=24,
min=8),
Option(name='channel_basic',
type='bool',
default=True,
desc='Share basic cluster information (size, version)'),
Option(name='channel_ident',
type='bool',
default=False,
desc='Share a user-provided description and/or contact email for the cluster'),
Option(name='channel_crash',
type='bool',
default=True,
desc='Share metadata about Ceph daemon crashes (version, stack straces, etc)'),
Option(name='channel_device',
type='bool',
default=True,
desc=('Share device health metrics '
'(e.g., SMART data, minus potentially identifying info like serial numbers)')),
Option(name='channel_perf',
type='bool',
default=False,
desc='Share perf counter metrics summed across the whole cluster'),
]
@property
def config_keys(self) -> Dict[str, OptionValue]:
return dict((o['name'], o.get('default', None)) for o in self.MODULE_OPTIONS)
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
self.event = Event()
self.run = False
self.last_upload: Optional[int] = None
self.last_report: Dict[str, Any] = dict()
self.report_id: Optional[str] = None
self.salt: Optional[str] = None
self.config_update_module_option()
# for mypy which does not run the code
if TYPE_CHECKING:
self.url = ''
self.device_url = ''
self.enabled = False
self.last_opt_revision = 0
self.leaderboard = ''
self.interval = 0
self.proxy = ''
self.channel_basic = True
self.channel_ident = False
self.channel_crash = True
self.channel_device = True
self.channel_perf = False
def config_update_module_option(self) -> None:
for opt in self.MODULE_OPTIONS:
setattr(self,
opt['name'],
self.get_module_option(opt['name']))
self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name']))
def config_notify(self) -> None:
self.config_update_module_option()
# wake up serve() thread
self.event.set()
def load(self) -> None:
last_upload = self.get_store('last_upload', None)
if last_upload is None:
self.last_upload = None
else:
self.last_upload = int(last_upload)
report_id = self.get_store('report_id', None)
if report_id is None:
self.report_id = str(uuid.uuid4())
self.set_store('report_id', self.report_id)
else:
self.report_id = report_id
salt = self.get_store('salt', None)
if salt is None:
self.salt = str(uuid.uuid4())
self.set_store('salt', self.salt)
else:
self.salt = salt
def gather_osd_metadata(self,
osd_map: Dict[str, List[Dict[str, int]]]) -> Dict[str, Dict[str, int]]:
keys = ["osd_objectstore", "rotational"]
keys += self.metadata_keys
metadata: Dict[str, Dict[str, int]] = dict()
for key in keys:
metadata[key] = defaultdict(int)
for osd in osd_map['osds']:
res = self.get_metadata('osd', str(osd['osd']))
if res is None:
self.log.debug('Could not get metadata for osd.%s' % str(osd['osd']))
continue
for k, v in res.items():
if k not in keys:
continue
metadata[k][v] += 1
return metadata
def gather_mon_metadata(self,
mon_map: Dict[str, List[Dict[str, str]]]) -> Dict[str, Dict[str, int]]:
keys = list()
keys += self.metadata_keys
metadata: Dict[str, Dict[str, int]] = dict()
for key in keys:
metadata[key] = defaultdict(int)
for mon in mon_map['mons']:
res = self.get_metadata('mon', mon['name'])
if res is None:
self.log.debug('Could not get metadata for mon.%s' % (mon['name']))
continue
for k, v in res.items():
if k not in keys:
continue
metadata[k][v] += 1
return metadata
def gather_crush_info(self) -> Dict[str, Union[int,
bool,
List[int],
Dict[str, int],
Dict[int, int]]]:
osdmap = self.get_osdmap()
crush_raw = osdmap.get_crush()
crush = crush_raw.dump()
BucketKeyT = TypeVar('BucketKeyT', int, str)
def inc(d: Dict[BucketKeyT, int], k: BucketKeyT) -> None:
if k in d:
d[k] += 1
else:
d[k] = 1
device_classes: Dict[str, int] = {}
for dev in crush['devices']:
inc(device_classes, dev.get('class', ''))
bucket_algs: Dict[str, int] = {}
bucket_types: Dict[str, int] = {}
bucket_sizes: Dict[int, int] = {}
for bucket in crush['buckets']:
if '~' in bucket['name']: # ignore shadow buckets
continue
inc(bucket_algs, bucket['alg'])
inc(bucket_types, bucket['type_id'])
inc(bucket_sizes, len(bucket['items']))
return {
'num_devices': len(crush['devices']),
'num_types': len(crush['types']),
'num_buckets': len(crush['buckets']),
'num_rules': len(crush['rules']),
'device_classes': list(device_classes.values()),
'tunables': crush['tunables'],
'compat_weight_set': '-1' in crush['choose_args'],
'num_weight_sets': len(crush['choose_args']),
'bucket_algs': bucket_algs,
'bucket_sizes': bucket_sizes,
'bucket_types': bucket_types,
}
def gather_configs(self) -> Dict[str, List[str]]:
# cluster config options
cluster = set()
r, outb, outs = self.mon_command({
'prefix': 'config dump',
'format': 'json'
})
if r != 0:
return {}
try:
dump = json.loads(outb)
except json.decoder.JSONDecodeError:
return {}
for opt in dump:
name = opt.get('name')
if name:
cluster.add(name)
# daemon-reported options (which may include ceph.conf)
active = set()
ls = self.get("modified_config_options")
for opt in ls.get('options', {}):
active.add(opt)
return {
'cluster_changed': sorted(list(cluster)),
'active_changed': sorted(list(active)),
}
def get_mempool(self, mode: str = 'separated') -> Dict[str, dict]:
# Initialize result dict
result: Dict[str, dict] = defaultdict(lambda: defaultdict(int))
# Get list of osd ids from the metadata
osd_metadata = self.get('osd_metadata')
# Grab output from the "osd.x dump_mempools" command
for osd_id in osd_metadata:
cmd_dict = {
'prefix': 'dump_mempools',
'id': str(osd_id),
'format': 'json'
}
r, outb, outs = self.osd_command(cmd_dict)
if r != 0:
self.log.debug("Invalid command dictionary.")
continue
else:
try:
# This is where the mempool will land.
dump = json.loads(outb)
if mode == 'separated':
result["osd." + str(osd_id)] = dump['mempool']['by_pool']
elif mode == 'aggregated':
for mem_type in dump['mempool']['by_pool']:
result[mem_type]['bytes'] += dump['mempool']['by_pool'][mem_type]['bytes']
result[mem_type]['items'] += dump['mempool']['by_pool'][mem_type]['items']
else:
self.log.debug("Incorrect mode specified in get_mempool")
except (json.decoder.JSONDecodeError, KeyError) as e:
self.log.debug("Error caught: {}".format(e))
return {}
return result
def get_osd_histograms(self, mode: str = 'separated') -> List[Dict[str, dict]]:
# Initialize result dict
result: Dict[str, dict] = defaultdict(lambda: defaultdict(
lambda: defaultdict(
lambda: defaultdict(
lambda: defaultdict(
lambda: defaultdict(int))))))
# Get list of osd ids from the metadata
osd_metadata = self.get('osd_metadata')
# Grab output from the "osd.x perf histogram dump" command
for osd_id in osd_metadata:
cmd_dict = {
'prefix': 'perf histogram dump',
'id': str(osd_id),
'format': 'json'
}
r, outb, outs = self.osd_command(cmd_dict)
# Check for invalid calls
if r != 0:
self.log.debug("Invalid command dictionary.")
continue
else:
try:
# This is where the histograms will land if there are any.
dump = json.loads(outb)
for histogram in dump['osd']:
# Log axis information. There are two axes, each represented
# as a dictionary. Both dictionaries are contained inside a
# list called 'axes'.
axes = []
for axis in dump['osd'][histogram]['axes']:
# This is the dict that contains information for an individual
# axis. It will be appended to the 'axes' list at the end.
axis_dict: Dict[str, Any] = defaultdict()
# Collecting information for buckets, min, name, etc.
axis_dict['buckets'] = axis['buckets']
axis_dict['min'] = axis['min']
axis_dict['name'] = axis['name']
axis_dict['quant_size'] = axis['quant_size']
axis_dict['scale_type'] = axis['scale_type']
# Collecting ranges; placing them in lists to
# improve readability later on.
ranges = []
for _range in axis['ranges']:
_max, _min = None, None
if 'max' in _range:
_max = _range['max']
if 'min' in _range:
_min = _range['min']
ranges.append([_min, _max])
axis_dict['ranges'] = ranges
# Now that 'axis_dict' contains all the appropriate
# information for the current axis, append it to the 'axes' list.
# There will end up being two axes in the 'axes' list, since the
# histograms are 2D.
axes.append(axis_dict)
# Add the 'axes' list, containing both axes, to result.
# At this point, you will see that the name of the key is the string
# form of our axes list (str(axes)). This is there so that histograms
# with different axis configs will not be combined.
# These key names are later dropped when only the values are returned.
result[str(axes)][histogram]['axes'] = axes
# Collect current values and make sure they are in
# integer form.
values = []
for value_list in dump['osd'][histogram]['values']:
values.append([int(v) for v in value_list])
if mode == 'separated':
if 'osds' not in result[str(axes)][histogram]:
result[str(axes)][histogram]['osds'] = []
result[str(axes)][histogram]['osds'].append({'osd_id': int(osd_id), 'values': values})
elif mode == 'aggregated':
# Aggregate values. If 'values' have already been initialized,
# we can safely add.
if 'values' in result[str(axes)][histogram]:
for i in range (0, len(values)):
for j in range (0, len(values[i])):
values[i][j] += result[str(axes)][histogram]['values'][i][j]
# Add the values to result.
result[str(axes)][histogram]['values'] = values
# Update num_combined_osds
if 'num_combined_osds' not in result[str(axes)][histogram]:
result[str(axes)][histogram]['num_combined_osds'] = 1
else:
result[str(axes)][histogram]['num_combined_osds'] += 1
else:
self.log.error('Incorrect mode specified in get_osd_histograms: {}'.format(mode))
return list()
# Sometimes, json errors occur if you give it an empty string.
# I am also putting in a catch for a KeyError since it could
# happen where the code is assuming that a key exists in the
# schema when it doesn't. In either case, we'll handle that
# by returning an empty dict.
except (json.decoder.JSONDecodeError, KeyError) as e:
self.log.debug("Error caught: {}".format(e))
return list()
return list(result.values())
def get_io_rate(self) -> dict:
return self.get('io_rate')
def gather_crashinfo(self) -> List[Dict[str, str]]:
crashlist: List[Dict[str, str]] = list()
errno, crashids, err = self.remote('crash', 'ls')
if errno:
return crashlist
for crashid in crashids.split():
errno, crashinfo, err = self.remote('crash', 'do_info', crashid)
if errno:
continue
c = json.loads(crashinfo)
# redact hostname
del c['utsname_hostname']
# entity_name might have more than one '.', beware
(etype, eid) = c.get('entity_name', '').split('.', 1)
m = hashlib.sha1()
assert self.salt
m.update(self.salt.encode('utf-8'))
m.update(eid.encode('utf-8'))
m.update(self.salt.encode('utf-8'))
c['entity_name'] = etype + '.' + m.hexdigest()
# redact final line of python tracebacks, as the exception
# payload may contain identifying information
if 'mgr_module' in c and 'backtrace' in c:
# backtrace might be empty
if len(c['backtrace']) > 0:
c['backtrace'][-1] = '<redacted>'
crashlist.append(c)
return crashlist
def gather_perf_counters(self, mode: str = 'separated') -> Dict[str, dict]:
# Extract perf counter data with get_all_perf_counters(), a method
# from mgr/mgr_module.py. This method returns a nested dictionary that
# looks a lot like perf schema, except with some additional fields.
#
# Example of output, a snapshot of a mon daemon:
# "mon.b": {
# "bluestore.kv_flush_lat": {
# "count": 2431,
# "description": "Average kv_thread flush latency",
# "nick": "fl_l",
# "priority": 8,
# "type": 5,
# "units": 1,
# "value": 88814109
# },
# },
all_perf_counters = self.get_all_perf_counters()
# Initialize 'result' dict
result: Dict[str, dict] = defaultdict(lambda: defaultdict(
lambda: defaultdict(lambda: defaultdict(int))))
for daemon in all_perf_counters:
# Calculate num combined daemon types if in aggregated mode
if mode == 'aggregated':
daemon_type = daemon[0:3] # i.e. 'mds', 'osd', 'rgw'
if 'num_combined_daemons' not in result[daemon_type]:
result[daemon_type]['num_combined_daemons'] = 1
else:
result[daemon_type]['num_combined_daemons'] += 1
for collection in all_perf_counters[daemon]:
# Split the collection to avoid redundancy in final report; i.e.:
# bluestore.kv_flush_lat, bluestore.kv_final_lat -->
# bluestore: kv_flush_lat, kv_final_lat
col_0, col_1 = collection.split('.')
# Debug log for empty keys. This initially was a problem for prioritycache
# perf counters, where the col_0 was empty for certain mon counters:
#
# "mon.a": { instead of "mon.a": {
# "": { "prioritycache": {
# "cache_bytes": {...}, "cache_bytes": {...},
#
# This log is here to detect any future instances of a similar issue.
if (daemon == "") or (col_0 == "") or (col_1 == ""):
self.log.debug("Instance of an empty key: {}{}".format(daemon, collection))
if mode == 'separated':
# Add value to result
result[daemon][col_0][col_1]['value'] = \
all_perf_counters[daemon][collection]['value']
# Check that 'count' exists, as not all counters have a count field.
if 'count' in all_perf_counters[daemon][collection]:
result[daemon][col_0][col_1]['count'] = \
all_perf_counters[daemon][collection]['count']
elif mode == 'aggregated':
# Not every rgw daemon has the same schema. Specifically, each rgw daemon
# has a uniquely-named collection that starts off identically (i.e.
# "objecter-0x...") then diverges (i.e. "...55f4e778e140.op_rmw").
# This bit of code combines these unique counters all under one rgw instance.
# Without this check, the schema would remain separeted out in the final report.
if col_0[0:11] == "objecter-0x":
col_0 = "objecter-0x"
# Check that the value can be incremented. In some cases,
# the files are of type 'pair' (real-integer-pair, integer-integer pair).
# In those cases, the value is a dictionary, and not a number.
# i.e. throttle-msgr_dispatch_throttler-hbserver["wait"]
if isinstance(all_perf_counters[daemon][collection]['value'], numbers.Number):
result[daemon_type][col_0][col_1]['value'] += \
all_perf_counters[daemon][collection]['value']
# Check that 'count' exists, as not all counters have a count field.
if 'count' in all_perf_counters[daemon][collection]:
result[daemon_type][col_0][col_1]['count'] += \
all_perf_counters[daemon][collection]['count']
else:
self.log.error('Incorrect mode specified in gather_perf_counters: {}'.format(mode))
return {}
return result
def get_active_channels(self) -> List[str]:
r = []
if self.channel_basic:
r.append('basic')
if self.channel_crash:
r.append('crash')
if self.channel_device:
r.append('device')
if self.channel_ident:
r.append('ident')
if self.channel_perf:
r.append('perf')
return r
def gather_device_report(self) -> Dict[str, Dict[str, Dict[str, str]]]:
try:
time_format = self.remote('devicehealth', 'get_time_format')
except Exception:
return {}
cutoff = datetime.utcnow() - timedelta(hours=self.interval * 2)
min_sample = cutoff.strftime(time_format)
devices = self.get('devices')['devices']
# anon-host-id -> anon-devid -> { timestamp -> record }
res: Dict[str, Dict[str, Dict[str, str]]] = {}
for d in devices:
devid = d['devid']
try:
# this is a map of stamp -> {device info}
m = self.remote('devicehealth', 'get_recent_device_metrics',
devid, min_sample)
except Exception:
continue
# anonymize host id
try:
host = d['location'][0]['host']
except (KeyError, IndexError):
continue
anon_host = self.get_store('host-id/%s' % host)
if not anon_host:
anon_host = str(uuid.uuid1())
self.set_store('host-id/%s' % host, anon_host)
serial = None
for dev, rep in m.items():
rep['host_id'] = anon_host
if serial is None and 'serial_number' in rep:
serial = rep['serial_number']
# anonymize device id
anon_devid = self.get_store('devid-id/%s' % devid)
if not anon_devid:
# ideally devid is 'vendor_model_serial',
# but can also be 'model_serial', 'serial'
if '_' in devid:
anon_devid = f"{devid.rsplit('_', 1)[0]}_{uuid.uuid1()}"
else:
anon_devid = str(uuid.uuid1())
self.set_store('devid-id/%s' % devid, anon_devid)
self.log.info('devid %s / %s, host %s / %s' % (devid, anon_devid,
host, anon_host))
# anonymize the smartctl report itself
if serial:
m_str = json.dumps(m)
m = json.loads(m_str.replace(serial, 'deleted'))
if anon_host not in res:
res[anon_host] = {}
res[anon_host][anon_devid] = m
return res
def get_latest(self, daemon_type: str, daemon_name: str, stat: str) -> int:
data = self.get_counter(daemon_type, daemon_name, stat)[stat]
if data:
return data[-1][1]
else:
return 0
def compile_report(self, channels: Optional[List[str]] = None) -> Dict[str, Any]:
if not channels:
channels = self.get_active_channels()
report = {
'leaderboard': self.leaderboard,
'report_version': 1,
'report_timestamp': datetime.utcnow().isoformat(),
'report_id': self.report_id,
'channels': channels,
'channels_available': ALL_CHANNELS,
'license': LICENSE,
}
if 'ident' in channels:
for option in ['description', 'contact', 'organization']:
report[option] = getattr(self, option)
if 'basic' in channels:
mon_map = self.get('mon_map')
osd_map = self.get('osd_map')
service_map = self.get('service_map')
fs_map = self.get('fs_map')
df = self.get('df')
report['created'] = mon_map['created']
# mons
v1_mons = 0
v2_mons = 0
ipv4_mons = 0
ipv6_mons = 0
for mon in mon_map['mons']:
for a in mon['public_addrs']['addrvec']:
if a['type'] == 'v2':
v2_mons += 1
elif a['type'] == 'v1':
v1_mons += 1
if a['addr'].startswith('['):
ipv6_mons += 1
else:
ipv4_mons += 1
report['mon'] = {
'count': len(mon_map['mons']),
'features': mon_map['features'],
'min_mon_release': mon_map['min_mon_release'],
'v1_addr_mons': v1_mons,
'v2_addr_mons': v2_mons,
'ipv4_addr_mons': ipv4_mons,
'ipv6_addr_mons': ipv6_mons,
}
report['config'] = self.gather_configs()
# pools
rbd_num_pools = 0
rbd_num_images_by_pool = []
rbd_mirroring_by_pool = []
num_pg = 0
report['pools'] = list()
for pool in osd_map['pools']:
num_pg += pool['pg_num']
ec_profile = {}
if pool['erasure_code_profile']:
orig = osd_map['erasure_code_profiles'].get(
pool['erasure_code_profile'], {})
ec_profile = {
k: orig[k] for k in orig.keys()
if k in ['k', 'm', 'plugin', 'technique',
'crush-failure-domain', 'l']
}
cast(List[Dict[str, Any]], report['pools']).append(
{
'pool': pool['pool'],
'pg_num': pool['pg_num'],
'pgp_num': pool['pg_placement_num'],
'size': pool['size'],
'min_size': pool['min_size'],
'pg_autoscale_mode': pool['pg_autoscale_mode'],
'target_max_bytes': pool['target_max_bytes'],
'target_max_objects': pool['target_max_objects'],
'type': ['', 'replicated', '', 'erasure'][pool['type']],
'erasure_code_profile': ec_profile,
'cache_mode': pool['cache_mode'],
}
)
if 'rbd' in pool['application_metadata']:
rbd_num_pools += 1
ioctx = self.rados.open_ioctx(pool['pool_name'])
rbd_num_images_by_pool.append(
sum(1 for _ in rbd.RBD().list2(ioctx)))
rbd_mirroring_by_pool.append(
rbd.RBD().mirror_mode_get(ioctx) != rbd.RBD_MIRROR_MODE_DISABLED)
report['rbd'] = {
'num_pools': rbd_num_pools,
'num_images_by_pool': rbd_num_images_by_pool,
'mirroring_by_pool': rbd_mirroring_by_pool}
# osds
cluster_network = False
for osd in osd_map['osds']:
if osd['up'] and not cluster_network:
front_ip = osd['public_addrs']['addrvec'][0]['addr'].split(':')[0]
back_ip = osd['cluster_addrs']['addrvec'][0]['addr'].split(':')[0]
if front_ip != back_ip:
cluster_network = True
report['osd'] = {
'count': len(osd_map['osds']),
'require_osd_release': osd_map['require_osd_release'],
'require_min_compat_client': osd_map['require_min_compat_client'],
'cluster_network': cluster_network,
}
# crush
report['crush'] = self.gather_crush_info()
# cephfs
report['fs'] = {
'count': len(fs_map['filesystems']),
'feature_flags': fs_map['feature_flags'],
'num_standby_mds': len(fs_map['standbys']),
'filesystems': [],
}
num_mds = len(fs_map['standbys'])
for fsm in fs_map['filesystems']:
fs = fsm['mdsmap']
num_sessions = 0
cached_ino = 0
cached_dn = 0
cached_cap = 0
subtrees = 0
rfiles = 0
rbytes = 0
rsnaps = 0
for gid, mds in fs['info'].items():
num_sessions += self.get_latest('mds', mds['name'],
'mds_sessions.session_count')
cached_ino += self.get_latest('mds', mds['name'],
'mds_mem.ino')
cached_dn += self.get_latest('mds', mds['name'],
'mds_mem.dn')
cached_cap += self.get_latest('mds', mds['name'],
'mds_mem.cap')
subtrees += self.get_latest('mds', mds['name'],
'mds.subtrees')
if mds['rank'] == 0:
rfiles = self.get_latest('mds', mds['name'],
'mds.root_rfiles')
rbytes = self.get_latest('mds', mds['name'],
'mds.root_rbytes')
rsnaps = self.get_latest('mds', mds['name'],
'mds.root_rsnaps')
report['fs']['filesystems'].append({ # type: ignore
'max_mds': fs['max_mds'],
'ever_allowed_features': fs['ever_allowed_features'],
'explicitly_allowed_features': fs['explicitly_allowed_features'],
'num_in': len(fs['in']),
'num_up': len(fs['up']),
'num_standby_replay': len(
[mds for gid, mds in fs['info'].items()
if mds['state'] == 'up:standby-replay']),
'num_mds': len(fs['info']),
'num_sessions': num_sessions,
'cached_inos': cached_ino,
'cached_dns': cached_dn,
'cached_caps': cached_cap,
'cached_subtrees': subtrees,
'balancer_enabled': len(fs['balancer']) > 0,
'num_data_pools': len(fs['data_pools']),
'standby_count_wanted': fs['standby_count_wanted'],
'approx_ctime': fs['created'][0:7],
'files': rfiles,
'bytes': rbytes,
'snaps': rsnaps,
})
num_mds += len(fs['info'])
report['fs']['total_num_mds'] = num_mds # type: ignore
# daemons
report['metadata'] = dict(osd=self.gather_osd_metadata(osd_map),
mon=self.gather_mon_metadata(mon_map))
# host counts
servers = self.list_servers()
self.log.debug('servers %s' % servers)
hosts = {
'num': len([h for h in servers if h['hostname']]),
}
for t in ['mon', 'mds', 'osd', 'mgr']:
nr_services = sum(1 for host in servers if
any(service for service in cast(List[ServiceInfoT],
host['services'])
if service['type'] == t))
hosts['num_with_' + t] = nr_services
report['hosts'] = hosts
report['usage'] = {
'pools': len(df['pools']),
'pg_num': num_pg,
'total_used_bytes': df['stats']['total_used_bytes'],
'total_bytes': df['stats']['total_bytes'],
'total_avail_bytes': df['stats']['total_avail_bytes']
}
services: DefaultDict[str, int] = defaultdict(int)
for key, value in service_map['services'].items():
services[key] += 1
if key == 'rgw':
rgw = {}
zones = set()
zonegroups = set()
frontends = set()
count = 0
d = value.get('daemons', dict())
for k, v in d.items():
if k == 'summary' and v:
rgw[k] = v
elif isinstance(v, dict) and 'metadata' in v:
count += 1
zones.add(v['metadata']['zone_id'])
zonegroups.add(v['metadata']['zonegroup_id'])
frontends.add(v['metadata']['frontend_type#0'])
# we could actually iterate over all the keys of
# the dict and check for how many frontends there
# are, but it is unlikely that one would be running
# more than 2 supported ones
f2 = v['metadata'].get('frontend_type#1', None)
if f2:
frontends.add(f2)
rgw['count'] = count
rgw['zones'] = len(zones)
rgw['zonegroups'] = len(zonegroups)
rgw['frontends'] = list(frontends) # sets aren't json-serializable
report['rgw'] = rgw
report['services'] = services
try:
report['balancer'] = self.remote('balancer', 'gather_telemetry')
except ImportError:
report['balancer'] = {
'active': False
}
if 'crash' in channels:
report['crashes'] = self.gather_crashinfo()
if 'perf' in channels:
report['perf_counters'] = self.gather_perf_counters('separated')
report['stats_per_pool'] = self.get('pg_dump')['pool_stats']
report['stats_per_pg'] = self.get('pg_dump')['pg_stats']
report['io_rate'] = self.get_io_rate()
report['osd_perf_histograms'] = self.get_osd_histograms('separated')
report['mempool'] = self.get_mempool('separated')
# NOTE: We do not include the 'device' channel in this report; it is
# sent to a different endpoint.
return report
def _try_post(self, what: str, url: str, report: Dict[str, Dict[str, str]]) -> Optional[str]:
self.log.info('Sending %s to: %s' % (what, url))
proxies = dict()
if self.proxy:
self.log.info('Send using HTTP(S) proxy: %s', self.proxy)
proxies['http'] = self.proxy
proxies['https'] = self.proxy
try:
resp = requests.put(url=url, json=report, proxies=proxies)
resp.raise_for_status()
except Exception as e:
fail_reason = 'Failed to send %s to %s: %s' % (what, url, str(e))
self.log.error(fail_reason)
return fail_reason
return None
class EndPoint(enum.Enum):
ceph = 'ceph'
device = 'device'
def send(self,
report: Dict[str, Dict[str, str]],
endpoint: Optional[List[EndPoint]] = None) -> Tuple[int, str, str]:
if not endpoint:
endpoint = [self.EndPoint.ceph, self.EndPoint.device]
failed = []
success = []
self.log.debug('Send endpoints %s' % endpoint)
for e in endpoint:
if e == self.EndPoint.ceph:
fail_reason = self._try_post('ceph report', self.url, report)
if fail_reason:
failed.append(fail_reason)
else:
now = int(time.time())
self.last_upload = now
self.set_store('last_upload', str(now))
success.append('Ceph report sent to {0}'.format(self.url))
self.log.info('Sent report to {0}'.format(self.url))
elif e == self.EndPoint.device:
if 'device' in self.get_active_channels():
devices = self.gather_device_report()
assert devices
num_devs = 0
num_hosts = 0
for host, ls in devices.items():
self.log.debug('host %s devices %s' % (host, ls))
if not len(ls):
continue
fail_reason = self._try_post('devices', self.device_url,
ls)
if fail_reason:
failed.append(fail_reason)
else:
num_devs += len(ls)
num_hosts += 1
if num_devs:
success.append('Reported %d devices across %d hosts' % (
num_devs, len(devices)))
if failed:
return 1, '', '\n'.join(success + failed)
return 0, '', '\n'.join(success)
@CLIReadCommand('telemetry status')
def status(self) -> Tuple[int, str, str]:
'''
Show current configuration
'''
r = {}
for opt in self.MODULE_OPTIONS:
r[opt['name']] = getattr(self, opt['name'])
r['last_upload'] = (time.ctime(self.last_upload)
if self.last_upload else self.last_upload)
return 0, json.dumps(r, indent=4, sort_keys=True), ''
@CLICommand('telemetry on')
def on(self, license: Optional[str] = None) -> Tuple[int, str, str]:
'''
Enable telemetry reports from this cluster
'''
if license != LICENSE:
return -errno.EPERM, '', f'''Telemetry data is licensed under the {LICENSE_NAME} ({LICENSE_URL}).
To enable, add '--license {LICENSE}' to the 'ceph telemetry on' command.'''
else:
self.set_module_option('enabled', True)
self.set_module_option('last_opt_revision', REVISION)
return 0, '', ''
@CLICommand('telemetry off')
def off(self) -> Tuple[int, str, str]:
'''
Disable telemetry reports from this cluster
'''
self.set_module_option('enabled', False)
self.set_module_option('last_opt_revision', 1)
return 0, '', ''
@CLICommand('telemetry send')
def do_send(self,
endpoint: Optional[List[EndPoint]] = None,
license: Optional[str] = None) -> Tuple[int, str, str]:
if self.last_opt_revision < LAST_REVISION_RE_OPT_IN and license != LICENSE:
self.log.debug(('A telemetry send attempt while opted-out. '
'Asking for license agreement'))
return -errno.EPERM, '', f'''Telemetry data is licensed under the {LICENSE_NAME} ({LICENSE_URL}).
To manually send telemetry data, add '--license {LICENSE}' to the 'ceph telemetry send' command.
Please consider enabling the telemetry module with 'ceph telemetry on'.'''
else:
self.last_report = self.compile_report()
return self.send(self.last_report, endpoint)
@CLIReadCommand('telemetry show')
def show(self, channels: Optional[List[str]] = None) -> Tuple[int, str, str]:
'''
Show report of all channels
'''
report = self.get_report(channels=channels)
# Formatting the perf histograms so they are human-readable. This will change the
# ranges and values, which are currently in list form, into strings so that
# they are displayed horizontally instead of vertically.
try:
# Formatting ranges and values in osd_perf_histograms
modes_to_be_formatted = ['osd_perf_histograms']
for mode in modes_to_be_formatted:
for config in report[mode]:
for histogram in config:
# Adjust ranges by converting lists into strings
for axis in config[histogram]['axes']:
for i in range(0, len(axis['ranges'])):
axis['ranges'][i] = str(axis['ranges'][i])
# Adjust values by converting lists into strings
if mode == 'osd_perf_histograms_aggregated':
for i in range(0, len(config[histogram]['values'])):
config[histogram]['values'][i] = str(config[histogram]['values'][i])
else: # if mode == 'osd_perf_histograms_separated'
for osd in config[histogram]['osds']:
for i in range(0, len(osd['values'])):
osd['values'][i] = str(osd['values'][i])
except KeyError:
# If the perf channel is not enabled, there should be a KeyError since
# 'osd_perf_histograms' would not be present in the report. In that case,
# the show function should pass as usual without trying to format the
# histograms.
pass
report = json.dumps(report, indent=4, sort_keys=True)
if self.channel_device:
report += '''
Device report is generated separately. To see it run 'ceph telemetry show-device'.'''
return 0, report, ''
@CLIReadCommand('telemetry show-device')
def show_device(self) -> Tuple[int, str, str]:
return 0, json.dumps(self.get_report('device'), indent=4, sort_keys=True), ''
@CLIReadCommand('telemetry show-all')
def show_all(self) -> Tuple[int, str, str]:
return 0, json.dumps(self.get_report('all'), indent=4, sort_keys=True), ''
def get_report(self,
report_type: str = 'default',
channels: Optional[List[str]] = None) -> Dict[str, Any]:
if report_type == 'default':
return self.compile_report(channels=channels)
elif report_type == 'device':
return self.gather_device_report()
elif report_type == 'all':
return {'report': self.compile_report(channels=channels),
'device_report': self.gather_device_report()}
return {}
def self_test(self) -> None:
report = self.compile_report()
if len(report) == 0:
raise RuntimeError('Report is empty')
if 'report_id' not in report:
raise RuntimeError('report_id not found in report')
def shutdown(self) -> None:
self.run = False
self.event.set()
def refresh_health_checks(self) -> None:
health_checks = {}
if self.enabled and self.last_opt_revision < LAST_REVISION_RE_OPT_IN:
health_checks['TELEMETRY_CHANGED'] = {
'severity': 'warning',
'summary': 'Telemetry requires re-opt-in',
'detail': [
'telemetry report includes new information; must re-opt-in (or out)'
]
}
self.set_health_checks(health_checks)
def serve(self) -> None:
self.load()
self.run = True
self.log.debug('Waiting for mgr to warm up')
time.sleep(10)
while self.run:
self.event.clear()
self.refresh_health_checks()
if self.last_opt_revision < LAST_REVISION_RE_OPT_IN:
self.log.debug('Not sending report until user re-opts-in')
self.event.wait(1800)
continue
if not self.enabled:
self.log.debug('Not sending report until configured to do so')
self.event.wait(1800)
continue
now = int(time.time())
if not self.last_upload or \
(now - self.last_upload) > self.interval * 3600:
self.log.info('Compiling and sending report to %s',
self.url)
try:
self.last_report = self.compile_report()
except Exception:
self.log.exception('Exception while compiling report:')
self.send(self.last_report)
else:
self.log.debug('Interval for sending new report has not expired')
sleep = 3600
self.log.debug('Sleeping for %d seconds', sleep)
self.event.wait(sleep)
@staticmethod
def can_run() -> Tuple[bool, str]:
return True, ''
class Module(MgrModule, orchestrator.Orchestrator):
"""An Orchestrator that uses <Ansible Runner Service> to perform operations
"""
MODULE_OPTIONS = [
# url:port of the Ansible Runner Service
Option(name="server_location", type="str", default=""),
# Check server identity (True by default)
Option(name="verify_server", type="bool", default=True),
# Path to an alternative CA bundle
Option(name="ca_bundle", type="str", default="")
]
def __init__(self, *args, **kwargs):
super(Module, self).__init__(*args, **kwargs)
self.run = False
self.all_completions = []
self._ar_client = None # type: Optional[Client]
# TLS certificate and key file names used to connect with the external
# Ansible Runner Service
self.client_cert_fname = ""
self.client_key_fname = ""
# used to provide more verbose explanation of errors in status method
self.status_message = ""
self.all_progress_references = list(
) # type: List[orchestrator.ProgressReference]
@property
def ar_client(self):
# type: () -> Client
assert self._ar_client is not None
return self._ar_client
def available(self):
""" Check if Ansible Runner service is working
"""
available = False
msg = ""
try:
if self._ar_client:
available = self.ar_client.is_operative()
if not available:
msg = "No response from Ansible Runner Service"
else:
msg = "Not possible to initialize connection with Ansible "\
"Runner service."
except AnsibleRunnerServiceError as ex:
available = False
msg = str(ex)
# Add more details to the detected problem
if self.status_message:
msg = "{}:\n{}".format(msg, self.status_message)
return (available, msg)
def process(self, completions):
"""Given a list of Completion instances, progress any which are
incomplete.
:param completions: list of Completion instances
:Returns : True if everything is done.
"""
if completions:
self.log.info("process: completions={0}".format(
orchestrator.pretty_print(completions)))
def serve(self):
""" Mandatory for standby modules
"""
self.log.info("Starting Ansible Orchestrator module ...")
try:
# Verify config options and client certificates
self.verify_config()
# Ansible runner service client
self._ar_client = Client(
server_url=self.get_module_option('server_location', ''),
verify_server=self.get_module_option('verify_server', True),
ca_bundle=self.get_module_option('ca_bundle', ''),
client_cert=self.client_cert_fname,
client_key=self.client_key_fname)
except AnsibleRunnerServiceError:
self.log.exception(
"Ansible Runner Service not available. "
"Check external server status/TLS identity or "
"connection options. If configuration options changed"
" try to disable/enable the module.")
self.shutdown()
return
self.run = True
def shutdown(self):
self.log.info('Stopping Ansible orchestrator module')
self.run = False
def get_inventory(self, node_filter=None, refresh=False):
"""
:param : node_filter instance
:param : refresh any cached state
:Return : A AnsibleReadOperation instance (Completion Object)
"""
# Create a new read completion object for execute the playbook
op = playbook_operation(client=self.ar_client,
playbook=GET_STORAGE_DEVICES_CATALOG_PLAYBOOK,
result_pattern="list storage inventory",
params={},
output_wizard=ProcessInventory(self.ar_client),
event_filter_list=["runner_on_ok"])
self._launch_operation(op)
return op
def create_osds(self, drive_group):
"""Create one or more OSDs within a single Drive Group.
If no host provided the operation affects all the host in the OSDS role
:param drive_group: (ceph.deployment.drive_group.DriveGroupSpec),
Drive group with the specification of drives to use
"""
# Transform drive group specification to Ansible playbook parameters
host, osd_spec = dg_2_ansible(drive_group)
# Create a new read completion object for execute the playbook
op = playbook_operation(client=self.ar_client,
playbook=ADD_OSD_PLAYBOOK,
result_pattern="",
params=osd_spec,
querystr_dict={"limit": host},
output_wizard=ProcessPlaybookResult(
self.ar_client),
event_filter_list=["playbook_on_stats"])
self._launch_operation(op)
return op
def remove_osds(self, osd_ids, destroy=False):
"""Remove osd's.
:param osd_ids: List of osd's to be removed (List[int])
:param destroy: unsupported.
"""
assert not destroy
extravars = {
'osd_to_kill': ",".join([str(osd_id) for osd_id in osd_ids]),
'ireallymeanit': 'yes'
}
# Create a new read completion object for execute the playbook
op = playbook_operation(client=self.ar_client,
playbook=REMOVE_OSD_PLAYBOOK,
result_pattern="",
params=extravars,
output_wizard=ProcessPlaybookResult(
self.ar_client),
event_filter_list=["playbook_on_stats"])
# Execute the playbook
self._launch_operation(op)
return op
def get_hosts(self):
"""Provides a list Inventory nodes
"""
host_ls_op = ars_read(self.ar_client,
url=URL_GET_HOSTS,
output_wizard=ProcessHostsList(self.ar_client))
return host_ls_op
def add_host(self, host):
"""
Add a host to the Ansible Runner Service inventory in the "orchestrator"
group
:param host: hostname
:returns : orchestrator.Completion
"""
url_group = URL_MANAGE_GROUP.format(group_name=ORCHESTRATOR_GROUP)
try:
# Create the orchestrator default group if not exist.
# If exists we ignore the error response
dummy_response = self.ar_client.http_post(url_group, "", {})
# Here, the default group exists so...
# Prepare the operation for adding the new host
add_url = URL_ADD_RM_HOSTS.format(
host_name=host, inventory_group=ORCHESTRATOR_GROUP)
operations = [ars_http_operation(add_url, "post", "", None)]
except AnsibleRunnerServiceError as ex:
# Problems with the external orchestrator.
# Prepare the operation to return the error in a Completion object.
self.log.exception("Error checking <orchestrator> group: %s",
str(ex))
operations = [ars_http_operation(url_group, "post", "", None)]
return ars_change(self.ar_client, operations)
def remove_host(self, host):
"""
Remove a host from all the groups in the Ansible Runner Service
inventory.
:param host: hostname
:returns : orchestrator.Completion
"""
host_groups = [] # type: List[Any]
try:
# Get the list of groups where the host is included
groups_url = URL_GET_HOST_GROUPS.format(host_name=host)
response = self.ar_client.http_get(groups_url)
if response.status_code == requests.codes.ok:
host_groups = json.loads(response.text)["data"]["groups"]
except AnsibleRunnerServiceError:
self.log.exception("Error retrieving host groups")
raise
if not host_groups:
# Error retrieving the groups, prepare the completion object to
# execute the problematic operation just to provide the error
# to the caller
operations = [ars_http_operation(groups_url, "get")]
else:
# Build the operations list
operations = list(
map(
lambda x: ars_http_operation(
URL_ADD_RM_HOSTS.format(host_name=host,
inventory_group=x), "delete"),
host_groups))
return ars_change(self.ar_client, operations)
def add_rgw(self, spec):
# type: (orchestrator.RGWSpec) -> orchestrator.Completion
""" Add a RGW service in the cluster
: spec : an Orchestrator.RGWSpec object
: returns : Completion object
"""
# Add the hosts to the inventory in the right group
hosts = spec.placement.hosts
if not hosts:
raise orchestrator.OrchestratorError(
"No hosts provided. "
"At least one destination host is needed to install the RGW "
"service")
def set_rgwspec_defaults(spec):
spec.rgw_multisite = spec.rgw_multisite if spec.rgw_multisite is not None else True
spec.rgw_zonemaster = spec.rgw_zonemaster if spec.rgw_zonemaster is not None else True
spec.rgw_zonesecondary = spec.rgw_zonesecondary \
if spec.rgw_zonesecondary is not None else False
spec.rgw_multisite_proto = spec.rgw_multisite_proto \
if spec.rgw_multisite_proto is not None else "http"
spec.rgw_frontend_port = spec.rgw_frontend_port \
if spec.rgw_frontend_port is not None else 8080
spec.rgw_zonegroup = spec.rgw_zonegroup if spec.rgw_zonegroup is not None else "default"
spec.rgw_zone_user = spec.rgw_zone_user if spec.rgw_zone_user is not None else "zone.user"
spec.rgw_realm = spec.rgw_realm if spec.rgw_realm is not None else "default"
spec.system_access_key = spec.system_access_key \
if spec.system_access_key is not None else spec.genkey(20)
spec.system_secret_key = spec.system_secret_key \
if spec.system_secret_key is not None else spec.genkey(40)
set_rgwspec_defaults(spec)
InventoryGroup("rgws", self.ar_client).update(hosts)
# Limit playbook execution to certain hosts
limited = ",".join(str(host) for host in hosts)
# Add the settings for this service
extravars = {
k: v
for (k, v) in spec.__dict__.items() if k.startswith('rgw_')
}
extravars['rgw_zone'] = spec.name
extravars[
'rgw_multisite_endpoint_addr'] = spec.rgw_multisite_endpoint_addr
extravars[
'rgw_multisite_endpoints_list'] = spec.rgw_multisite_endpoints_list
extravars['rgw_frontend_port'] = str(spec.rgw_frontend_port)
# Group hosts by resource (used in rm ops)
resource_group = "rgw_zone_{}".format(spec.name)
InventoryGroup(resource_group, self.ar_client).update(hosts)
# Execute the playbook to create the service
op = playbook_operation(client=self.ar_client,
playbook=SITE_PLAYBOOK,
result_pattern="",
params=extravars,
querystr_dict={"limit": limited},
output_wizard=ProcessPlaybookResult(
self.ar_client),
event_filter_list=["playbook_on_stats"])
# Execute the playbook
self._launch_operation(op)
return op
def remove_rgw(self, zone):
""" Remove a RGW service providing <zone>
:param zone: <zone name> of the RGW
...
:returns : Completion object
"""
# Ansible Inventory group for the kind of service
group = "rgws"
# get the list of hosts where to remove the service
# (hosts in resource group)
resource_group = "rgw_zone_{}".format(zone)
hosts_list = list(InventoryGroup(resource_group, self.ar_client))
limited = ",".join(hosts_list)
# Avoid manual confirmation
extravars = {"ireallymeanit": "yes"}
# Cleaning of inventory after a sucessful operation
clean_inventory = {}
clean_inventory[resource_group] = hosts_list
clean_inventory[group] = hosts_list
# Execute the playbook to remove the service
op = playbook_operation(client=self.ar_client,
playbook=PURGE_PLAYBOOK,
result_pattern="",
params=extravars,
querystr_dict={"limit": limited},
output_wizard=ProcessPlaybookResult(
self.ar_client),
event_filter_list=["playbook_on_stats"],
clean_hosts_on_success=clean_inventory)
# Execute the playbook
self._launch_operation(op)
return op
def _launch_operation(self, ansible_operation):
"""Launch the operation and add the operation to the completion objects
ongoing
:ansible_operation: A read/write ansible operation (completion object)
"""
# Add the operation to the list of things ongoing
self.all_completions.append(ansible_operation)
def verify_config(self):
"""Verify mandatory settings for the module and provide help to
configure properly the orchestrator
"""
# Retrieve TLS content to use and check them
# First try to get certiticate and key content for this manager instance
# ex: mgr/ansible/mgr0/[crt/key]
self.log.info("Tying to use configured specific certificate and key"
"files for this server")
the_crt = self.get_store("{}/{}".format(self.get_mgr_id(), "crt"))
the_key = self.get_store("{}/{}".format(self.get_mgr_id(), "key"))
if the_crt is None or the_key is None:
# If not possible... try to get generic certificates and key content
# ex: mgr/ansible/[crt/key]
self.log.warning("Specific tls files for this manager not "
"configured, trying to use generic files")
the_crt = self.get_store("crt")
the_key = self.get_store("key")
if the_crt is None or the_key is None:
self.status_message = "No client certificate configured. Please "\
"set Ansible Runner Service client "\
"certificate and key:\n"\
"ceph ansible set-ssl-certificate-"\
"{key,certificate} -i <file>"
self.log.error(self.status_message)
return
# generate certificate temp files
self.client_cert_fname = generate_temp_file("crt", the_crt)
self.client_key_fname = generate_temp_file("key", the_key)
try:
verify_tls_files(self.client_cert_fname, self.client_key_fname)
except ServerConfigException as e:
self.status_message = str(e)
if self.status_message:
self.log.error(self.status_message)
return
# Check module options
if not self.get_module_option("server_location", ""):
self.status_message = "No Ansible Runner Service base URL "\
"<server_name>:<port>."\
"Try 'ceph config set mgr mgr/{0}/server_location "\
"<server name/ip>:<port>'".format(self.module_name)
self.log.error(self.status_message)
return
if self.get_module_option("verify_server", True):
self.status_message = "TLS server identity verification is enabled"\
" by default.Use 'ceph config set mgr mgr/{0}/verify_server False'"\
"to disable it.Use 'ceph config set mgr mgr/{0}/ca_bundle <path>'"\
"to point an alternative CA bundle path used for TLS server "\
"verification".format(self.module_name)
self.log.error(self.status_message)
return
# Everything ok
self.status_message = ""
#---------------------------------------------------------------------------
# Ansible Orchestrator self-owned commands
#---------------------------------------------------------------------------
@CLIWriteCommand("ansible set-ssl-certificate",
"name=mgr_id,type=CephString,req=false")
def set_tls_certificate(self, mgr_id=None, inbuf=None):
"""Load tls certificate in mon k-v store
"""
if inbuf is None:
return -errno.EINVAL, \
'Please specify the certificate file with "-i" option', ''
if mgr_id is not None:
self.set_store("{}/crt".format(mgr_id), inbuf)
else:
self.set_store("crt", inbuf)
return 0, "SSL certificate updated", ""
@CLIWriteCommand("ansible set-ssl-certificate-key",
"name=mgr_id,type=CephString,req=false")
def set_tls_certificate_key(self, mgr_id=None, inbuf=None):
"""Load tls certificate key in mon k-v store
"""
if inbuf is None:
return -errno.EINVAL, \
'Please specify the certificate key file with "-i" option', \
''
if mgr_id is not None:
self.set_store("{}/key".format(mgr_id), inbuf)
else:
self.set_store("key", inbuf)
return 0, "SSL certificate key updated", ""
class Module(MgrModule):
MODULE_OPTIONS = [
Option(
'allow_m_granularity',
type='bool',
default=False,
desc='allow minute scheduled snapshots',
runtime=True,
),
Option(
'dump_on_update',
type='bool',
default=False,
desc='dump database to debug log on update',
runtime=True,
),
]
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
self._initialized = Event()
self.client = SnapSchedClient(self)
def resolve_subvolume_path(self, fs: str, subvol: Optional[str], path: str) -> str:
if not subvol:
return path
rc, subvol_path, err = self.remote('fs', 'subvolume', 'getpath',
fs, subvol)
if rc != 0:
# TODO custom exception?
raise Exception(f'Could not resolve {path} in {fs}, {subvol}')
return subvol_path + path
@property
def default_fs(self) -> str:
fs_map = self.get('fs_map')
if fs_map['filesystems']:
return fs_map['filesystems'][0]['mdsmap']['fs_name']
else:
self.log.error('No filesystem instance could be found.')
raise CephfsConnectionException(
-errno.ENOENT, "no filesystem found")
def has_fs(self, fs_name: str) -> bool:
return fs_name in self.client.get_all_filesystems()
def serve(self) -> None:
self._initialized.set()
def handle_command(self, inbuf: str, cmd: Dict[str, str]) -> Tuple[int, str, str]:
self._initialized.wait()
return -errno.EINVAL, "", "Unknown command"
@CLIReadCommand('fs snap-schedule status')
def snap_schedule_get(self,
path: str = '/',
subvol: Optional[str] = None,
fs: Optional[str] = None,
format: Optional[str] = 'plain') -> Tuple[int, str, str]:
'''
List current snapshot schedules
'''
use_fs = fs if fs else self.default_fs
if not self.has_fs(use_fs):
return -errno.EINVAL, '', f"no such filesystem: {use_fs}"
try:
ret_scheds = self.client.get_snap_schedules(use_fs, path)
except CephfsConnectionException as e:
return e.to_tuple()
if format == 'json':
json_report = ','.join([ret_sched.report_json() for ret_sched in ret_scheds])
return 0, f'[{json_report}]', ''
return 0, '\n===\n'.join([ret_sched.report() for ret_sched in ret_scheds]), ''
@CLIReadCommand('fs snap-schedule list')
def snap_schedule_list(self, path: str,
subvol: Optional[str] = None,
recursive: bool = False,
fs: Optional[str] = None,
format: Optional[str] = 'plain') -> Tuple[int, str, str]:
'''
Get current snapshot schedule for <path>
'''
try:
use_fs = fs if fs else self.default_fs
if not self.has_fs(use_fs):
return -errno.EINVAL, '', f"no such filesystem: {use_fs}"
scheds = self.client.list_snap_schedules(use_fs, path, recursive)
self.log.debug(f'recursive is {recursive}')
except CephfsConnectionException as e:
return e.to_tuple()
if not scheds:
if format == 'json':
output: Dict[str, str] = {}
return 0, json.dumps(output), ''
return -errno.ENOENT, '', f'SnapSchedule for {path} not found'
if format == 'json':
# json_list = ','.join([sched.json_list() for sched in scheds])
schedule_list = [sched.schedule for sched in scheds]
retention_list = [sched.retention for sched in scheds]
out = {'path': path, 'schedule': schedule_list, 'retention': retention_list}
return 0, json.dumps(out), ''
return 0, '\n'.join([str(sched) for sched in scheds]), ''
@CLIWriteCommand('fs snap-schedule add')
def snap_schedule_add(self,
path: str,
snap_schedule: str,
start: Optional[str] = None,
fs: Optional[str] = None,
subvol: Optional[str] = None) -> Tuple[int, str, str]:
'''
Set a snapshot schedule for <path>
'''
try:
use_fs = fs if fs else self.default_fs
if not self.has_fs(use_fs):
return -errno.EINVAL, '', f"no such filesystem: {use_fs}"
abs_path = self.resolve_subvolume_path(use_fs, subvol, path)
self.client.store_snap_schedule(use_fs,
abs_path,
(abs_path, snap_schedule,
use_fs, path, start, subvol))
suc_msg = f'Schedule set for path {path}'
except sqlite3.IntegrityError:
existing_scheds = self.client.get_snap_schedules(use_fs, path)
report = [s.report() for s in existing_scheds]
error_msg = f'Found existing schedule {report}'
self.log.error(error_msg)
return -errno.EEXIST, '', error_msg
except ValueError as e:
return -errno.ENOENT, '', str(e)
except CephfsConnectionException as e:
return e.to_tuple()
return 0, suc_msg, ''
@CLIWriteCommand('fs snap-schedule remove')
def snap_schedule_rm(self,
path: str,
repeat: Optional[str] = None,
start: Optional[str] = None,
subvol: Optional[str] = None,
fs: Optional[str] = None) -> Tuple[int, str, str]:
'''
Remove a snapshot schedule for <path>
'''
try:
use_fs = fs if fs else self.default_fs
if not self.has_fs(use_fs):
return -errno.EINVAL, '', f"no such filesystem: {use_fs}"
abs_path = self.resolve_subvolume_path(use_fs, subvol, path)
self.client.rm_snap_schedule(use_fs, abs_path, repeat, start)
except CephfsConnectionException as e:
return e.to_tuple()
except ValueError as e:
return -errno.ENOENT, '', str(e)
return 0, 'Schedule removed for path {}'.format(path), ''
@CLIWriteCommand('fs snap-schedule retention add')
def snap_schedule_retention_add(self,
path: str,
retention_spec_or_period: str,
retention_count: Optional[str] = None,
fs: Optional[str] = None,
subvol: Optional[str] = None) -> Tuple[int, str, str]:
'''
Set a retention specification for <path>
'''
try:
use_fs = fs if fs else self.default_fs
if not self.has_fs(use_fs):
return -errno.EINVAL, '', f"no such filesystem: {use_fs}"
abs_path = self.resolve_subvolume_path(use_fs, subvol, path)
self.client.add_retention_spec(use_fs, abs_path,
retention_spec_or_period,
retention_count)
except CephfsConnectionException as e:
return e.to_tuple()
except ValueError as e:
return -errno.ENOENT, '', str(e)
return 0, 'Retention added to path {}'.format(path), ''
@CLIWriteCommand('fs snap-schedule retention remove')
def snap_schedule_retention_rm(self,
path: str,
retention_spec_or_period: str,
retention_count: Optional[str] = None,
fs: Optional[str] = None,
subvol: Optional[str] = None) -> Tuple[int, str, str]:
'''
Remove a retention specification for <path>
'''
try:
use_fs = fs if fs else self.default_fs
if not self.has_fs(use_fs):
return -errno.EINVAL, '', f"no such filesystem: {use_fs}"
abs_path = self.resolve_subvolume_path(use_fs, subvol, path)
self.client.rm_retention_spec(use_fs, abs_path,
retention_spec_or_period,
retention_count)
except CephfsConnectionException as e:
return e.to_tuple()
except ValueError as e:
return -errno.ENOENT, '', str(e)
return 0, 'Retention removed from path {}'.format(path), ''
@CLIWriteCommand('fs snap-schedule activate')
def snap_schedule_activate(self,
path: str,
repeat: Optional[str] = None,
start: Optional[str] = None,
subvol: Optional[str] = None,
fs: Optional[str] = None) -> Tuple[int, str, str]:
'''
Activate a snapshot schedule for <path>
'''
try:
use_fs = fs if fs else self.default_fs
if not self.has_fs(use_fs):
return -errno.EINVAL, '', f"no such filesystem: {use_fs}"
abs_path = self.resolve_subvolume_path(use_fs, subvol, path)
self.client.activate_snap_schedule(use_fs, abs_path, repeat, start)
except CephfsConnectionException as e:
return e.to_tuple()
except ValueError as e:
return -errno.ENOENT, '', str(e)
return 0, 'Schedule activated for path {}'.format(path), ''
@CLIWriteCommand('fs snap-schedule deactivate')
def snap_schedule_deactivate(self,
path: str,
repeat: Optional[str] = None,
start: Optional[str] = None,
subvol: Optional[str] = None,
fs: Optional[str] = None) -> Tuple[int, str, str]:
'''
Deactivate a snapshot schedule for <path>
'''
try:
use_fs = fs if fs else self.default_fs
if not self.has_fs(use_fs):
return -errno.EINVAL, '', f"no such filesystem: {use_fs}"
abs_path = self.resolve_subvolume_path(use_fs, subvol, path)
self.client.deactivate_snap_schedule(use_fs, abs_path, repeat, start)
except CephfsConnectionException as e:
return e.to_tuple()
except ValueError as e:
return -errno.ENOENT, '', str(e)
return 0, 'Schedule deactivated for path {}'.format(path), ''
class Module(MgrModule):
MODULE_OPTIONS = [
Option(name=MirrorSnapshotScheduleHandler.MODULE_OPTION_NAME),
Option(name=MirrorSnapshotScheduleHandler.MODULE_OPTION_NAME_MAX_CONCURRENT_SNAP_CREATE,
type='int',
default=10),
Option(name=TrashPurgeScheduleHandler.MODULE_OPTION_NAME),
]
def __init__(self, *args: Any, **kwargs: Any) -> None:
super(Module, self).__init__(*args, **kwargs)
self.rados.wait_for_latest_osdmap()
self.mirror_snapshot_schedule = MirrorSnapshotScheduleHandler(self)
self.perf = PerfHandler(self)
self.task = TaskHandler(self)
self.trash_purge_schedule = TrashPurgeScheduleHandler(self)
@CLIWriteCommand('rbd mirror snapshot schedule add')
@with_latest_osdmap
def mirror_snapshot_schedule_add(self,
level_spec: str,
interval: str,
start_time: Optional[str] = None) -> Tuple[int, str, str]:
"""
Add rbd mirror snapshot schedule
"""
spec = LevelSpec.from_name(self, level_spec, namespace_validator, image_validator)
return self.mirror_snapshot_schedule.add_schedule(spec, interval, start_time)
@CLIWriteCommand('rbd mirror snapshot schedule remove')
@with_latest_osdmap
def mirror_snapshot_schedule_remove(self,
level_spec: str,
interval: Optional[str] = None,
start_time: Optional[str] = None) -> Tuple[int, str, str]:
"""
Remove rbd mirror snapshot schedule
"""
spec = LevelSpec.from_name(self, level_spec, namespace_validator, image_validator)
return self.mirror_snapshot_schedule.remove_schedule(spec, interval, start_time)
@CLIReadCommand('rbd mirror snapshot schedule list')
@with_latest_osdmap
def mirror_snapshot_schedule_list(self,
level_spec: str = '') -> Tuple[int, str, str]:
"""
List rbd mirror snapshot schedule
"""
spec = LevelSpec.from_name(self, level_spec, namespace_validator, image_validator)
return self.mirror_snapshot_schedule.list(spec)
@CLIReadCommand('rbd mirror snapshot schedule status')
@with_latest_osdmap
def mirror_snapshot_schedule_status(self,
level_spec: str = '') -> Tuple[int, str, str]:
"""
Show rbd mirror snapshot schedule status
"""
spec = LevelSpec.from_name(self, level_spec, namespace_validator, image_validator)
return self.mirror_snapshot_schedule.status(spec)
@CLIReadCommand('rbd perf image stats')
@with_latest_osdmap
def perf_image_stats(self,
pool_spec: Optional[str] = None,
sort_by: Optional[ImageSortBy] = None) -> Tuple[int, str, str]:
"""
Retrieve current RBD IO performance stats
"""
with self.perf.lock:
sort_by_name = sort_by.name if sort_by else OSD_PERF_QUERY_COUNTERS[0]
return self.perf.get_perf_stats(pool_spec, sort_by_name)
@CLIReadCommand('rbd perf image counters')
@with_latest_osdmap
def perf_image_counters(self,
pool_spec: Optional[str] = None,
sort_by: Optional[ImageSortBy] = None) -> Tuple[int, str, str]:
"""
Retrieve current RBD IO performance counters
"""
with self.perf.lock:
sort_by_name = sort_by.name if sort_by else OSD_PERF_QUERY_COUNTERS[0]
return self.perf.get_perf_counters(pool_spec, sort_by_name)
@CLIWriteCommand('rbd task add flatten')
@with_latest_osdmap
def task_add_flatten(self, image_spec: str) -> Tuple[int, str, str]:
"""
Flatten a cloned image asynchronously in the background
"""
with self.task.lock:
return self.task.queue_flatten(image_spec)
@CLIWriteCommand('rbd task add remove')
@with_latest_osdmap
def task_add_remove(self, image_spec: str) -> Tuple[int, str, str]:
"""
Remove an image asynchronously in the background
"""
with self.task.lock:
return self.task.queue_remove(image_spec)
@CLIWriteCommand('rbd task add trash remove')
@with_latest_osdmap
def task_add_trash_remove(self, image_id_spec: str) -> Tuple[int, str, str]:
"""
Remove an image from the trash asynchronously in the background
"""
with self.task.lock:
return self.task.queue_trash_remove(image_id_spec)
@CLIWriteCommand('rbd task add migration execute')
@with_latest_osdmap
def task_add_migration_execute(self, image_spec: str) -> Tuple[int, str, str]:
"""
Execute an image migration asynchronously in the background
"""
with self.task.lock:
return self.task.queue_migration_execute(image_spec)
@CLIWriteCommand('rbd task add migration commit')
@with_latest_osdmap
def task_add_migration_commit(self, image_spec: str) -> Tuple[int, str, str]:
"""
Commit an executed migration asynchronously in the background
"""
with self.task.lock:
return self.task.queue_migration_commit(image_spec)
@CLIWriteCommand('rbd task add migration abort')
@with_latest_osdmap
def task_add_migration_abort(self, image_spec: str) -> Tuple[int, str, str]:
"""
Abort a prepared migration asynchronously in the background
"""
with self.task.lock:
return self.task.queue_migration_abort(image_spec)
@CLIWriteCommand('rbd task cancel')
@with_latest_osdmap
def task_cancel(self, task_id: str) -> Tuple[int, str, str]:
"""
Cancel a pending or running asynchronous task
"""
with self.task.lock:
return self.task.task_cancel(task_id)
@CLIReadCommand('rbd task list')
@with_latest_osdmap
def task_list(self, task_id: Optional[str] = None) -> Tuple[int, str, str]:
"""
List pending or running asynchronous tasks
"""
with self.task.lock:
return self.task.task_list(task_id)
@CLIWriteCommand('rbd trash purge schedule add')
@with_latest_osdmap
def trash_purge_schedule_add(self,
level_spec: str,
interval: str,
start_time: Optional[str] = None) -> Tuple[int, str, str]:
"""
Add rbd trash purge schedule
"""
spec = LevelSpec.from_name(self, level_spec, allow_image_level=False)
return self.trash_purge_schedule.add_schedule(spec, interval, start_time)
@CLIWriteCommand('rbd trash purge schedule remove')
@with_latest_osdmap
def trash_purge_schedule_remove(self,
level_spec: str,
interval: Optional[str] = None,
start_time: Optional[str] = None) -> Tuple[int, str, str]:
"""
Remove rbd trash purge schedule
"""
spec = LevelSpec.from_name(self, level_spec, allow_image_level=False)
return self.trash_purge_schedule.remove_schedule(spec, interval, start_time)
@CLIReadCommand('rbd trash purge schedule list')
@with_latest_osdmap
def trash_purge_schedule_list(self,
level_spec: str = '') -> Tuple[int, str, str]:
"""
List rbd trash purge schedule
"""
spec = LevelSpec.from_name(self, level_spec, allow_image_level=False)
return self.trash_purge_schedule.list(spec)
@CLIReadCommand('rbd trash purge schedule status')
@with_latest_osdmap
def trash_purge_schedule_status(self,
level_spec: str = '') -> Tuple[int, str, str]:
"""
Show rbd trash purge schedule status
"""
spec = LevelSpec.from_name(self, level_spec, allow_image_level=False)
return self.trash_purge_schedule.status(spec)
