def get_options(self): return [ Option( name=self.OPTION_FMT.format(feature), default=(feature not in PREDISABLED_FEATURES), type='bool', ) for feature in Features ]
class PgAutoscaler(MgrModule): """ PG autoscaler. """ NATIVE_OPTIONS = [ 'mon_target_pg_per_osd', 'mon_max_pg_per_osd', ] MODULE_OPTIONS = [ Option(name='sleep_interval', type='secs', default=60), Option( name='threshold', type='float', desc='scaling threshold', long_desc= ('The factor by which the `NEW PG_NUM` must vary from the current' '`PG_NUM` before being accepted. Cannot be less than 1.0'), default=3.0, min=1.0), ] def __init__(self, *args: Any, **kwargs: Any) -> None: super(PgAutoscaler, self).__init__(*args, **kwargs) self._shutdown = threading.Event() self._event: Dict[int, PgAdjustmentProgress] = {} # So much of what we do peeks at the osdmap that it's easiest # to just keep a copy of the pythonized version. self._osd_map = None if TYPE_CHECKING: self.sleep_interval = 60 self.mon_target_pg_per_osd = 0 self.threshold = 3.0 def config_notify(self) -> None: for opt in self.NATIVE_OPTIONS: setattr(self, opt, self.get_ceph_option(opt)) self.log.debug(' native option %s = %s', opt, getattr(self, opt)) for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' mgr option %s = %s', opt['name'], getattr(self, opt['name'])) @CLIReadCommand('osd pool autoscale-status') def _command_autoscale_status(self, format: str = 'plain' ) -> Tuple[int, str, str]: """ report on pool pg_num sizing recommendation and intent """ osdmap = self.get_osdmap() pools = osdmap.get_pools_by_name() ps, root_map = self._get_pool_status(osdmap, pools) if format in ('json', 'json-pretty'): return 0, json.dumps(ps, indent=4, sort_keys=True), '' else: table = PrettyTable( [ 'POOL', 'SIZE', 'TARGET SIZE', 'RATE', 'RAW CAPACITY', 'RATIO', 'TARGET RATIO', 'EFFECTIVE RATIO', 'BIAS', 'PG_NUM', # 'IDEAL', 'NEW PG_NUM', 'AUTOSCALE', 'BULK' ], border=False) table.left_padding_width = 0 table.right_padding_width = 2 table.align['POOL'] = 'l' table.align['SIZE'] = 'r' table.align['TARGET SIZE'] = 'r' table.align['RATE'] = 'r' table.align['RAW CAPACITY'] = 'r' table.align['RATIO'] = 'r' table.align['TARGET RATIO'] = 'r' table.align['EFFECTIVE RATIO'] = 'r' table.align['BIAS'] = 'r' table.align['PG_NUM'] = 'r' # table.align['IDEAL'] = 'r' table.align['NEW PG_NUM'] = 'r' table.align['AUTOSCALE'] = 'l' table.align['BULK'] = 'l' for p in ps: if p['would_adjust']: final = str(p['pg_num_final']) else: final = '' if p['target_bytes'] > 0: ts = mgr_util.format_bytes(p['target_bytes'], 6) else: ts = '' if p['target_ratio'] > 0.0: tr = '%.4f' % p['target_ratio'] else: tr = '' if p['effective_target_ratio'] > 0.0: etr = '%.4f' % p['effective_target_ratio'] else: etr = '' table.add_row([ p['pool_name'], mgr_util.format_bytes(p['logical_used'], 6), ts, p['raw_used_rate'], mgr_util.format_bytes(p['subtree_capacity'], 6), '%.4f' % p['capacity_ratio'], tr, etr, p['bias'], p['pg_num_target'], # p['pg_num_ideal'], final, p['pg_autoscale_mode'], str(p['bulk']) ]) return 0, table.get_string(), '' @CLIWriteCommand("osd pool set threshold") def set_scaling_threshold(self, num: float) -> Tuple[int, str, str]: """ set the autoscaler threshold A.K.A. the factor by which the new PG_NUM must vary from the existing PG_NUM """ if num < 1.0: return 22, "", "threshold cannot be set less than 1.0" self.set_module_option("threshold", num) return 0, "threshold updated", "" def serve(self) -> None: self.config_notify() while not self._shutdown.is_set(): self._maybe_adjust() self._update_progress_events() self._shutdown.wait(timeout=self.sleep_interval) def shutdown(self) -> None: self.log.info('Stopping pg_autoscaler') self._shutdown.set() def identify_subtrees_and_overlaps(self, osdmap: OSDMap, crush: CRUSHMap, result: Dict[int, CrushSubtreeResourceStatus], overlapped_roots: Set[int], roots: List[CrushSubtreeResourceStatus]) -> \ Tuple[List[CrushSubtreeResourceStatus], Set[int]]: # We identify subtrees and overlapping roots from osdmap for pool_id, pool in osdmap.get_pools().items(): crush_rule = crush.get_rule_by_id(pool['crush_rule']) assert crush_rule is not None cr_name = crush_rule['rule_name'] root_id = crush.get_rule_root(cr_name) assert root_id is not None osds = set(crush.get_osds_under(root_id)) # Are there overlapping roots? s = None for prev_root_id, prev in result.items(): if osds & prev.osds: s = prev if prev_root_id != root_id: overlapped_roots.add(prev_root_id) overlapped_roots.add(root_id) self.log.error('pool %d has overlapping roots: %s', pool_id, overlapped_roots) break if not s: s = CrushSubtreeResourceStatus() roots.append(s) result[root_id] = s s.root_ids.append(root_id) s.osds |= osds s.pool_ids.append(pool_id) s.pool_names.append(pool['pool_name']) s.pg_current += pool['pg_num_target'] * pool['size'] target_ratio = pool['options'].get('target_size_ratio', 0.0) if target_ratio: s.total_target_ratio += target_ratio else: target_bytes = pool['options'].get('target_size_bytes', 0) if target_bytes: s.total_target_bytes += target_bytes * osdmap.pool_raw_used_rate( pool_id) return roots, overlapped_roots def get_subtree_resource_status( self, osdmap: OSDMap, crush: CRUSHMap ) -> Tuple[Dict[int, CrushSubtreeResourceStatus], Set[int]]: """ For each CRUSH subtree of interest (i.e. the roots under which we have pools), calculate the current resource usages and targets, such as how many PGs there are, vs. how many PGs we would like there to be. """ result: Dict[int, CrushSubtreeResourceStatus] = {} roots: List[CrushSubtreeResourceStatus] = [] overlapped_roots: Set[int] = set() # identify subtrees and overlapping roots roots, overlapped_roots = self.identify_subtrees_and_overlaps( osdmap, crush, result, overlapped_roots, roots) # finish subtrees all_stats = self.get('osd_stats') for s in roots: assert s.osds is not None s.osd_count = len(s.osds) s.pg_target = s.osd_count * self.mon_target_pg_per_osd s.pg_left = s.pg_target s.pool_count = len(s.pool_ids) capacity = 0 for osd_stats in all_stats['osd_stats']: if osd_stats['osd'] in s.osds: # Intentionally do not apply the OSD's reweight to # this, because we want to calculate PG counts based # on the physical storage available, not how it is # reweighted right now. capacity += osd_stats['kb'] * 1024 s.capacity = capacity self.log.debug('root_ids %s pools %s with %d osds, pg_target %d', s.root_ids, s.pool_ids, s.osd_count, s.pg_target) return result, overlapped_roots def _calc_final_pg_target( self, p: Dict[str, Any], pool_name: str, root_map: Dict[int, CrushSubtreeResourceStatus], root_id: int, capacity_ratio: float, bias: float, even_pools: Dict[str, Dict[str, Any]], bulk_pools: Dict[str, Dict[str, Any]], func_pass: '******', bulk: bool, ) -> Union[Tuple[float, int, int], Tuple[None, None, None]]: """ `profile` determines behaviour of the autoscaler. `first_pass` flag used to determine if this is the first pass where the caller tries to calculate/adjust pools that has used_ratio > even_ratio else this is the second pass, we calculate final_ratio by giving it 1 / pool_count of the root we are currently looking at. """ if func_pass == 'first': # first pass to deal with small pools (no bulk flag) # calculating final_pg_target based on capacity ratio # we also keep track of bulk_pools to be used in second pass if not bulk: final_ratio = capacity_ratio pg_left = root_map[root_id].pg_left assert pg_left is not None used_pg = final_ratio * pg_left root_map[root_id].pg_left -= int(used_pg) root_map[root_id].pool_used += 1 pool_pg_target = used_pg / p['size'] * bias else: bulk_pools[pool_name] = p return None, None, None elif func_pass == 'second': # second pass we calculate the final_pg_target # for pools that have used_ratio > even_ratio # and we keep track of even pools to be used in third pass pool_count = root_map[root_id].pool_count assert pool_count is not None even_ratio = 1 / (pool_count - root_map[root_id].pool_used) used_ratio = capacity_ratio if used_ratio > even_ratio: root_map[root_id].pool_used += 1 else: even_pools[pool_name] = p return None, None, None final_ratio = max(used_ratio, even_ratio) pg_left = root_map[root_id].pg_left assert pg_left is not None used_pg = final_ratio * pg_left root_map[root_id].pg_left -= int(used_pg) pool_pg_target = used_pg / p['size'] * bias else: # third pass we just split the pg_left to all even_pools pool_count = root_map[root_id].pool_count assert pool_count is not None final_ratio = 1 / (pool_count - root_map[root_id].pool_used) pool_pg_target = (final_ratio * root_map[root_id].pg_left) / p['size'] * bias final_pg_target = max( p.get('options', {}).get('pg_num_min', PG_NUM_MIN), nearest_power_of_two(pool_pg_target)) self.log.info("Pool '{0}' root_id {1} using {2} of space, bias {3}, " "pg target {4} quantized to {5} (current {6})".format( p['pool_name'], root_id, capacity_ratio, bias, pool_pg_target, final_pg_target, p['pg_num_target'])) return final_ratio, pool_pg_target, final_pg_target def _get_pool_pg_targets( self, osdmap: OSDMap, pools: Dict[str, Dict[str, Any]], crush_map: CRUSHMap, root_map: Dict[int, CrushSubtreeResourceStatus], pool_stats: Dict[int, Dict[str, int]], ret: List[Dict[str, Any]], threshold: float, func_pass: '******', overlapped_roots: Set[int], ) -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]], Dict[str, Dict[ str, Any]]]: """ Calculates final_pg_target of each pools and determine if it needs scaling, this depends on the profile of the autoscaler. For scale-down, we start out with a full complement of pgs and only descrease it when other pools needs more pgs due to increased usage. For scale-up, we start out with the minimal amount of pgs and only scale when there is increase in usage. """ even_pools: Dict[str, Dict[str, Any]] = {} bulk_pools: Dict[str, Dict[str, Any]] = {} for pool_name, p in pools.items(): pool_id = p['pool'] if pool_id not in pool_stats: # race with pool deletion; skip continue # FIXME: we assume there is only one take per pool, but that # may not be true. crush_rule = crush_map.get_rule_by_id(p['crush_rule']) assert crush_rule is not None cr_name = crush_rule['rule_name'] root_id = crush_map.get_rule_root(cr_name) assert root_id is not None if root_id in overlapped_roots: # skip pools # with overlapping roots self.log.warn( "pool %d contains an overlapping root %d" "... skipping scaling", pool_id, root_id) continue capacity = root_map[root_id].capacity assert capacity is not None if capacity == 0: self.log.debug('skipping empty subtree %s', cr_name) continue raw_used_rate = osdmap.pool_raw_used_rate(pool_id) pool_logical_used = pool_stats[pool_id]['stored'] bias = p['options'].get('pg_autoscale_bias', 1.0) target_bytes = 0 # ratio takes precedence if both are set if p['options'].get('target_size_ratio', 0.0) == 0.0: target_bytes = p['options'].get('target_size_bytes', 0) # What proportion of space are we using? actual_raw_used = pool_logical_used * raw_used_rate actual_capacity_ratio = float(actual_raw_used) / capacity pool_raw_used = max(pool_logical_used, target_bytes) * raw_used_rate capacity_ratio = float(pool_raw_used) / capacity self.log.info("effective_target_ratio {0} {1} {2} {3}".format( p['options'].get('target_size_ratio', 0.0), root_map[root_id].total_target_ratio, root_map[root_id].total_target_bytes, capacity)) target_ratio = effective_target_ratio( p['options'].get('target_size_ratio', 0.0), root_map[root_id].total_target_ratio, root_map[root_id].total_target_bytes, capacity) # determine if the pool is a bulk bulk = False flags = p['flags_names'].split(",") if "bulk" in flags: bulk = True capacity_ratio = max(capacity_ratio, target_ratio) final_ratio, pool_pg_target, final_pg_target = self._calc_final_pg_target( p, pool_name, root_map, root_id, capacity_ratio, bias, even_pools, bulk_pools, func_pass, bulk) if final_ratio is None: continue adjust = False if (final_pg_target > p['pg_num_target'] * threshold or final_pg_target < p['pg_num_target'] / threshold) and \ final_ratio >= 0.0 and \ final_ratio <= 1.0: adjust = True assert pool_pg_target is not None ret.append({ 'pool_id': pool_id, 'pool_name': p['pool_name'], 'crush_root_id': root_id, 'pg_autoscale_mode': p['pg_autoscale_mode'], 'pg_num_target': p['pg_num_target'], 'logical_used': pool_logical_used, 'target_bytes': target_bytes, 'raw_used_rate': raw_used_rate, 'subtree_capacity': capacity, 'actual_raw_used': actual_raw_used, 'raw_used': pool_raw_used, 'actual_capacity_ratio': actual_capacity_ratio, 'capacity_ratio': capacity_ratio, 'target_ratio': p['options'].get('target_size_ratio', 0.0), 'effective_target_ratio': target_ratio, 'pg_num_ideal': int(pool_pg_target), 'pg_num_final': final_pg_target, 'would_adjust': adjust, 'bias': p.get('options', {}).get('pg_autoscale_bias', 1.0), 'bulk': bulk, }) return ret, bulk_pools, even_pools def _get_pool_status( self, osdmap: OSDMap, pools: Dict[str, Dict[str, Any]], ) -> Tuple[List[Dict[str, Any]], Dict[int, CrushSubtreeResourceStatus]]: threshold = self.threshold assert threshold >= 1.0 crush_map = osdmap.get_crush() root_map, overlapped_roots = self.get_subtree_resource_status( osdmap, crush_map) df = self.get('df') pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) ret: List[Dict[str, Any]] = [] # Iterate over all pools to determine how they should be sized. # First call of _get_pool_pg_targets() is to find/adjust pools that uses more capacaity than # the even_ratio of other pools and we adjust those first. # Second call make use of the even_pools we keep track of in the first call. # All we need to do is iterate over those and give them 1/pool_count of the # total pgs. ret, bulk_pools, _ = self._get_pool_pg_targets(osdmap, pools, crush_map, root_map, pool_stats, ret, threshold, 'first', overlapped_roots) ret, _, even_pools = self._get_pool_pg_targets(osdmap, bulk_pools, crush_map, root_map, pool_stats, ret, threshold, 'second', overlapped_roots) ret, _, _ = self._get_pool_pg_targets(osdmap, even_pools, crush_map, root_map, pool_stats, ret, threshold, 'third', overlapped_roots) return (ret, root_map) def _update_progress_events(self) -> None: osdmap = self.get_osdmap() pools = osdmap.get_pools() for pool_id in list(self._event): ev = self._event[pool_id] pool_data = pools.get(pool_id) if pool_data is None or pool_data['pg_num'] == pool_data[ 'pg_num_target'] or ev.pg_num == ev.pg_num_target: # pool is gone or we've reached our target self.remote('progress', 'complete', ev.ev_id) del self._event[pool_id] continue ev.update(self, (ev.pg_num - pool_data['pg_num']) / (ev.pg_num - ev.pg_num_target)) def _maybe_adjust(self) -> None: self.log.info('_maybe_adjust') osdmap = self.get_osdmap() if osdmap.get_require_osd_release() < 'nautilus': return pools = osdmap.get_pools_by_name() ps, root_map = self._get_pool_status(osdmap, pools) # Anyone in 'warn', set the health message for them and then # drop them from consideration. too_few = [] too_many = [] bytes_and_ratio = [] health_checks: Dict[str, Dict[str, Union[int, str, List[str]]]] = {} total_bytes = dict([(r, 0) for r in iter(root_map)]) total_target_bytes = dict([(r, 0.0) for r in iter(root_map)]) target_bytes_pools: Dict[int, List[int]] = dict([(r, []) for r in iter(root_map)]) for p in ps: pool_id = p['pool_id'] pool_opts = pools[p['pool_name']]['options'] if pool_opts.get('target_size_ratio', 0) > 0 and pool_opts.get( 'target_size_bytes', 0) > 0: bytes_and_ratio.append( 'Pool %s has target_size_bytes and target_size_ratio set' % p['pool_name']) total_bytes[p['crush_root_id']] += max( p['actual_raw_used'], p['target_bytes'] * p['raw_used_rate']) if p['target_bytes'] > 0: total_target_bytes[p[ 'crush_root_id']] += p['target_bytes'] * p['raw_used_rate'] target_bytes_pools[p['crush_root_id']].append(p['pool_name']) if not p['would_adjust']: continue if p['pg_autoscale_mode'] == 'warn': msg = 'Pool %s has %d placement groups, should have %d' % ( p['pool_name'], p['pg_num_target'], p['pg_num_final']) if p['pg_num_final'] > p['pg_num_target']: too_few.append(msg) else: too_many.append(msg) if p['pg_autoscale_mode'] == 'on': # Note that setting pg_num actually sets pg_num_target (see # OSDMonitor.cc) r = self.mon_command({ 'prefix': 'osd pool set', 'pool': p['pool_name'], 'var': 'pg_num', 'val': str(p['pg_num_final']) }) # create new event or update existing one to reflect # progress from current state to the new pg_num_target pool_data = pools[p['pool_name']] pg_num = pool_data['pg_num'] new_target = p['pg_num_final'] if pool_id in self._event: self._event[pool_id].reset(pg_num, new_target) else: self._event[pool_id] = PgAdjustmentProgress( pool_id, pg_num, new_target) self._event[pool_id].update(self, 0.0) if r[0] != 0: # FIXME: this is a serious and unexpected thing, # we should expose it as a cluster log error once # the hook for doing that from ceph-mgr modules is # in. self.log.error( "pg_num adjustment on {0} to {1} failed: {2}".format( p['pool_name'], p['pg_num_final'], r)) if too_few: summary = "{0} pools have too few placement groups".format( len(too_few)) health_checks['POOL_TOO_FEW_PGS'] = { 'severity': 'warning', 'summary': summary, 'count': len(too_few), 'detail': too_few } if too_many: summary = "{0} pools have too many placement groups".format( len(too_many)) health_checks['POOL_TOO_MANY_PGS'] = { 'severity': 'warning', 'summary': summary, 'count': len(too_many), 'detail': too_many } too_much_target_bytes = [] for root_id, total in total_bytes.items(): total_target = int(total_target_bytes[root_id]) capacity = root_map[root_id].capacity assert capacity is not None if total_target > 0 and total > capacity and capacity: too_much_target_bytes.append( 'Pools %s overcommit available storage by %.03fx due to ' 'target_size_bytes %s on pools %s' % (root_map[root_id].pool_names, total / capacity, mgr_util.format_bytes(total_target, 5, colored=False), target_bytes_pools[root_id])) elif total_target > capacity and capacity: too_much_target_bytes.append( 'Pools %s overcommit available storage by %.03fx due to ' 'collective target_size_bytes of %s' % ( root_map[root_id].pool_names, total / capacity, mgr_util.format_bytes(total_target, 5, colored=False), )) if too_much_target_bytes: health_checks['POOL_TARGET_SIZE_BYTES_OVERCOMMITTED'] = { 'severity': 'warning', 'summary': "%d subtrees have overcommitted pool target_size_bytes" % len(too_much_target_bytes), 'count': len(too_much_target_bytes), 'detail': too_much_target_bytes, } if bytes_and_ratio: health_checks['POOL_HAS_TARGET_SIZE_BYTES_AND_RATIO'] = { 'severity': 'warning', 'summary': "%d pools have both target_size_bytes and target_size_ratio set" % len(bytes_and_ratio), 'count': len(bytes_and_ratio), 'detail': bytes_and_ratio, } self.set_health_checks(health_checks)
class Module(MgrModule, CherryPyConfig): """ dashboard module entrypoint """ COMMANDS = [ { 'cmd': 'dashboard set-jwt-token-ttl ' 'name=seconds,type=CephInt', 'desc': 'Set the JWT token TTL in seconds', 'perm': 'w' }, { 'cmd': 'dashboard get-jwt-token-ttl', 'desc': 'Get the JWT token TTL in seconds', 'perm': 'r' }, { "cmd": "dashboard create-self-signed-cert", "desc": "Create self signed certificate", "perm": "w" }, ] COMMANDS.extend(options_command_list()) COMMANDS.extend(SSO_COMMANDS) PLUGIN_MANAGER.hook.register_commands() MODULE_OPTIONS = [ Option(name='server_addr', type='str', default='::'), Option(name='server_port', type='int', default=8443), Option(name='jwt_token_ttl', type='int', default=28800), Option(name='password', type='str', default=''), Option(name='url_prefix', type='str', default=''), Option(name='username', type='str', default=''), Option(name='key_file', type='str', default=''), Option(name='crt_file', type='str', default=''), Option(name='ssl', type='bool', default=True) ] MODULE_OPTIONS.extend(options_schema_list()) for options in PLUGIN_MANAGER.hook.get_options() or []: MODULE_OPTIONS.extend(options) __pool_stats = collections.defaultdict( lambda: collections.defaultdict(lambda: collections.deque(maxlen=10))) def __init__(self, *args, **kwargs): super(Module, self).__init__(*args, **kwargs) CherryPyConfig.__init__(self) mgr.init(self) self._stopping = threading.Event() self.shutdown_event = threading.Event() self.ACCESS_CTRL_DB = None self.SSO_DB = None @classmethod def can_run(cls): if cherrypy is None: return False, "Missing dependency: cherrypy" if not os.path.exists(cls.get_frontend_path()): return False, "Frontend assets not found: incomplete build?" return True, "" @classmethod def get_frontend_path(cls): current_dir = os.path.dirname(os.path.abspath(__file__)) return os.path.join(current_dir, 'frontend/dist') def serve(self): AuthManager.initialize() load_sso_db() uri = self.await_configuration() if uri is None: # We were shut down while waiting return # Publish the URI that others may use to access the service we're # about to start serving self.set_uri(uri) mapper, parent_urls = generate_routes(self.url_prefix) config = { self.url_prefix or '/': { 'tools.staticdir.on': True, 'tools.staticdir.dir': self.get_frontend_path(), 'tools.staticdir.index': 'index.html' } } for purl in parent_urls: config[purl] = {'request.dispatch': mapper} cherrypy.tree.mount(None, config=config) PLUGIN_MANAGER.hook.setup() cherrypy.engine.start() NotificationQueue.start_queue() TaskManager.init() logger.info('Engine started.') # wait for the shutdown event self.shutdown_event.wait() self.shutdown_event.clear() NotificationQueue.stop() cherrypy.engine.stop() logger.info('Engine stopped') def shutdown(self): super(Module, self).shutdown() CherryPyConfig.shutdown(self) logger.info('Stopping engine...') self.shutdown_event.set() def handle_command(self, inbuf, cmd): # pylint: disable=too-many-return-statements res = handle_option_command(cmd) if res[0] != -errno.ENOSYS: return res res = handle_sso_command(cmd) if res[0] != -errno.ENOSYS: return res if cmd['prefix'] == 'dashboard set-jwt-token-ttl': self.set_module_option('jwt_token_ttl', str(cmd['seconds'])) return 0, 'JWT token TTL updated', '' if cmd['prefix'] == 'dashboard get-jwt-token-ttl': ttl = self.get_module_option('jwt_token_ttl', JwtManager.JWT_TOKEN_TTL) return 0, str(ttl), '' if cmd['prefix'] == 'dashboard create-self-signed-cert': self.create_self_signed_cert() return 0, 'Self-signed certificate created', '' return (-errno.EINVAL, '', 'Command not found \'{0}\''.format(cmd['prefix'])) def create_self_signed_cert(self): # create a key pair pkey = crypto.PKey() pkey.generate_key(crypto.TYPE_RSA, 2048) # create a self-signed cert cert = crypto.X509() cert.get_subject().O = "IT" cert.get_subject().CN = "ceph-dashboard" cert.set_serial_number(int(uuid4())) cert.gmtime_adj_notBefore(0) cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) cert.set_issuer(cert.get_subject()) cert.set_pubkey(pkey) cert.sign(pkey, 'sha512') cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) self.set_store('crt', cert.decode('utf-8')) pkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey) self.set_store('key', pkey.decode('utf-8')) def notify(self, notify_type, notify_id): NotificationQueue.new_notification(notify_type, notify_id) def get_updated_pool_stats(self): df = self.get('df') pool_stats = {p['id']: p['stats'] for p in df['pools']} now = time.time() for pool_id, stats in pool_stats.items(): for stat_name, stat_val in stats.items(): self.__pool_stats[pool_id][stat_name].append((now, stat_val)) return self.__pool_stats
class Module(MgrModule): run = False config: Dict[str, OptionValue] = {} ceph_health_mapping = {'HEALTH_OK': 0, 'HEALTH_WARN': 1, 'HEALTH_ERR': 2} _zabbix_hosts: List[Dict[str, Union[str, int]]] = list() @property def config_keys(self) -> Dict[str, OptionValue]: return dict( (o['name'], o.get('default', None)) for o in self.MODULE_OPTIONS) MODULE_OPTIONS = [ Option(name='zabbix_sender', default='/usr/bin/zabbix_sender'), Option(name='zabbix_host', type='str', default=None), Option(name='zabbix_port', type='int', default=10051), Option(name='identifier', default=""), Option(name='interval', type='secs', default=60), Option(name='discovery_interval', type='uint', default=100) ] COMMANDS = [ { "cmd": "zabbix config-set name=key,type=CephString " "name=value,type=CephString", "desc": "Set a configuration value", "perm": "rw" }, { "cmd": "zabbix config-show", "desc": "Show current configuration", "perm": "r" }, { "cmd": "zabbix send", "desc": "Force sending data to Zabbix", "perm": "rw" }, { "cmd": "zabbix discovery", "desc": "Discovering Zabbix data", "perm": "r" }, ] def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) self.event = Event() def init_module_config(self) -> None: self.fsid = self.get('mon_map')['fsid'] self.log.debug('Found Ceph fsid %s', self.fsid) for key, default in self.config_keys.items(): self.set_config_option(key, self.get_module_option(key, default)) if self.config['zabbix_host']: self._parse_zabbix_hosts() def set_config_option(self, option: str, value: OptionValue) -> bool: if option not in self.config_keys.keys(): raise RuntimeError('{0} is a unknown configuration ' 'option'.format(option)) if option in ['zabbix_port', 'interval', 'discovery_interval']: try: int_value = int(value) # type: ignore except (ValueError, TypeError): raise RuntimeError('invalid {0} configured. Please specify ' 'a valid integer'.format(option)) if option == 'interval' and int_value < 10: raise RuntimeError('interval should be set to at least 10 seconds') if option == 'discovery_interval' and int_value < 10: raise RuntimeError( "discovery_interval should not be more frequent " "than once in 10 regular data collection") self.log.debug('Setting in-memory config option %s to: %s', option, value) self.config[option] = value return True def _parse_zabbix_hosts(self) -> None: self._zabbix_hosts = list() servers = cast(str, self.config['zabbix_host']).split(",") for server in servers: uri = re.match( "(?:(?:\[?)([a-z0-9-\.]+|[a-f0-9:\.]+)(?:\]?))(?:((?::))([0-9]{1,5}))?$", server) if uri: zabbix_host, sep, opt_zabbix_port = uri.groups() if sep == ':': zabbix_port = int(opt_zabbix_port) else: zabbix_port = cast(int, self.config['zabbix_port']) self._zabbix_hosts.append({ 'zabbix_host': zabbix_host, 'zabbix_port': zabbix_port }) else: self.log.error('Zabbix host "%s" is not valid', server) self.log.error('Parsed Zabbix hosts: %s', self._zabbix_hosts) def get_pg_stats(self) -> Dict[str, int]: stats = dict() pg_states = [ 'active', 'peering', 'clean', 'scrubbing', 'undersized', 'backfilling', 'recovering', 'degraded', 'inconsistent', 'remapped', 'backfill_toofull', 'backfill_wait', 'recovery_wait' ] for state in pg_states: stats['num_pg_{0}'.format(state)] = 0 pg_status = self.get('pg_status') stats['num_pg'] = pg_status['num_pgs'] for state in pg_status['pgs_by_state']: states = state['state_name'].split('+') for s in pg_states: key = 'num_pg_{0}'.format(s) if s in states: stats[key] += state['count'] return stats def get_data(self) -> Dict[str, Union[int, float]]: data = dict() health = json.loads(self.get('health')['json']) # 'status' is luminous+, 'overall_status' is legacy mode. data['overall_status'] = health.get('status', health.get('overall_status')) data['overall_status_int'] = \ self.ceph_health_mapping.get(data['overall_status']) mon_status = json.loads(self.get('mon_status')['json']) data['num_mon'] = len(mon_status['monmap']['mons']) df = self.get('df') data['num_pools'] = len(df['pools']) data['total_used_bytes'] = df['stats']['total_used_bytes'] data['total_bytes'] = df['stats']['total_bytes'] data['total_avail_bytes'] = df['stats']['total_avail_bytes'] wr_ops = 0 rd_ops = 0 wr_bytes = 0 rd_bytes = 0 for pool in df['pools']: wr_ops += pool['stats']['wr'] rd_ops += pool['stats']['rd'] wr_bytes += pool['stats']['wr_bytes'] rd_bytes += pool['stats']['rd_bytes'] data['[{0},rd_bytes]'.format( pool['name'])] = pool['stats']['rd_bytes'] data['[{0},wr_bytes]'.format( pool['name'])] = pool['stats']['wr_bytes'] data['[{0},rd_ops]'.format(pool['name'])] = pool['stats']['rd'] data['[{0},wr_ops]'.format(pool['name'])] = pool['stats']['wr'] data['[{0},bytes_used]'.format( pool['name'])] = pool['stats']['bytes_used'] data['[{0},stored_raw]'.format( pool['name'])] = pool['stats']['stored_raw'] data['[{0},percent_used]'.format( pool['name'])] = pool['stats']['percent_used'] * 100 data['wr_ops'] = wr_ops data['rd_ops'] = rd_ops data['wr_bytes'] = wr_bytes data['rd_bytes'] = rd_bytes osd_map = self.get('osd_map') data['num_osd'] = len(osd_map['osds']) data['osd_nearfull_ratio'] = osd_map['nearfull_ratio'] data['osd_full_ratio'] = osd_map['full_ratio'] data['osd_backfillfull_ratio'] = osd_map['backfillfull_ratio'] data['num_pg_temp'] = len(osd_map['pg_temp']) num_up = 0 num_in = 0 for osd in osd_map['osds']: data['[osd.{0},up]'.format(int(osd['osd']))] = osd['up'] if osd['up'] == 1: num_up += 1 data['[osd.{0},in]'.format(int(osd['osd']))] = osd['in'] if osd['in'] == 1: num_in += 1 data['num_osd_up'] = num_up data['num_osd_in'] = num_in osd_fill = list() osd_pgs = list() osd_apply_latency_ns = list() osd_commit_latency_ns = list() osd_stats = self.get('osd_stats') for osd in osd_stats['osd_stats']: try: osd_fill.append( (float(osd['kb_used']) / float(osd['kb'])) * 100) data['[osd.{0},osd_fill]'.format( osd['osd'])] = (float(osd['kb_used']) / float(osd['kb'])) * 100 except ZeroDivisionError: continue osd_pgs.append(osd['num_pgs']) osd_apply_latency_ns.append(osd['perf_stat']['apply_latency_ns']) osd_commit_latency_ns.append(osd['perf_stat']['commit_latency_ns']) data['[osd.{0},num_pgs]'.format(osd['osd'])] = osd['num_pgs'] data['[osd.{0},osd_latency_apply]'.format( osd['osd'] )] = osd['perf_stat']['apply_latency_ns'] / 1000000.0 # ns -> ms data['[osd.{0},osd_latency_commit]'.format( osd['osd'] )] = osd['perf_stat']['commit_latency_ns'] / 1000000.0 # ns -> ms try: data['osd_max_fill'] = max(osd_fill) data['osd_min_fill'] = min(osd_fill) data['osd_avg_fill'] = avg(osd_fill) data['osd_max_pgs'] = max(osd_pgs) data['osd_min_pgs'] = min(osd_pgs) data['osd_avg_pgs'] = avg(osd_pgs) except ValueError: pass try: data['osd_latency_apply_max'] = max( osd_apply_latency_ns) / 1000000.0 # ns -> ms data['osd_latency_apply_min'] = min( osd_apply_latency_ns) / 1000000.0 # ns -> ms data['osd_latency_apply_avg'] = avg( osd_apply_latency_ns) / 1000000.0 # ns -> ms data['osd_latency_commit_max'] = max( osd_commit_latency_ns) / 1000000.0 # ns -> ms data['osd_latency_commit_min'] = min( osd_commit_latency_ns) / 1000000.0 # ns -> ms data['osd_latency_commit_avg'] = avg( osd_commit_latency_ns) / 1000000.0 # ns -> ms except ValueError: pass data.update(self.get_pg_stats()) return data def send(self, data: Mapping[str, Union[int, float, str]]) -> bool: identifier = cast(Optional[str], self.config['identifier']) if identifier is None or len(identifier) == 0: identifier = 'ceph-{0}'.format(self.fsid) if not self.config['zabbix_host'] or not self._zabbix_hosts: self.log.error('Zabbix server not set, please configure using: ' 'ceph zabbix config-set zabbix_host <zabbix_host>') self.set_health_checks({ 'MGR_ZABBIX_NO_SERVER': { 'severity': 'warning', 'summary': 'No Zabbix server configured', 'detail': ['Configuration value zabbix_host not configured'] } }) return False result = True for server in self._zabbix_hosts: self.log.info( 'Sending data to Zabbix server %s, port %s as host/identifier %s', server['zabbix_host'], server['zabbix_port'], identifier) self.log.debug(data) try: zabbix = ZabbixSender(cast(str, self.config['zabbix_sender']), cast(str, server['zabbix_host']), cast(int, server['zabbix_port']), self.log) zabbix.send(identifier, data) except Exception as exc: self.log.exception('Failed to send.') self.set_health_checks({ 'MGR_ZABBIX_SEND_FAILED': { 'severity': 'warning', 'summary': 'Failed to send data to Zabbix', 'detail': [str(exc)] } }) result = False self.set_health_checks(dict()) return result def discovery(self) -> bool: osd_map = self.get('osd_map') osd_map_crush = self.get('osd_map_crush') # Discovering ceph pools pool_discovery = { pool['pool_name']: step['item_name'] for pool in osd_map['pools'] for rule in osd_map_crush['rules'] if rule['rule_id'] == pool['crush_rule'] for step in rule['steps'] if step['op'] == "take" } pools_discovery_data = { "data": [{ "{#POOL}": pool, "{#CRUSH_RULE}": rule } for pool, rule in pool_discovery.items()] } # Discovering OSDs # Getting hosts for found crush rules osd_roots = { step['item_name']: [item['id'] for item in root_bucket['items']] for rule in osd_map_crush['rules'] for step in rule['steps'] if step['op'] == "take" for root_bucket in osd_map_crush['buckets'] if root_bucket['id'] == step['item'] } # Getting osds for hosts with map to crush_rule osd_discovery = { item['id']: crush_rule for crush_rule, roots in osd_roots.items() for root in roots for bucket in osd_map_crush['buckets'] if bucket['id'] == root for item in bucket['items'] } osd_discovery_data = { "data": [{ "{#OSD}": osd, "{#CRUSH_RULE}": rule } for osd, rule in osd_discovery.items()] } # Preparing recieved data for sending data = { "zabbix.pool.discovery": json.dumps(pools_discovery_data), "zabbix.osd.discovery": json.dumps(osd_discovery_data) } return bool(self.send(data)) @CLIReadCommand('zabbix config-show') def config_show(self) -> Tuple[int, str, str]: return 0, json.dumps(self.config, indent=4, sort_keys=True), '' @CLIWriteCommand('zabbix config-set') def config_set(self, key: str, value: str) -> Tuple[int, str, str]: if not value: return -errno.EINVAL, '', 'Value should not be empty or None' self.log.debug('Setting configuration option %s to %s', key, value) if self.set_config_option(key, value): self.set_module_option(key, value) if key == 'zabbix_host' or key == 'zabbix_port': self._parse_zabbix_hosts() return 0, 'Configuration option {0} updated'.format(key), '' return 1,\ 'Failed to update configuration option {0}'.format(key), '' @CLIReadCommand('zabbix send') def do_send(self) -> Tuple[int, str, str]: data = self.get_data() if self.send(data): return 0, 'Sending data to Zabbix', '' return 1, 'Failed to send data to Zabbix', '' @CLIReadCommand('zabbix discovery') def do_discovery(self) -> Tuple[int, str, str]: if self.discovery(): return 0, 'Sending discovery data to Zabbix', '' return 1, 'Failed to send discovery data to Zabbix', '' def shutdown(self) -> None: self.log.info('Stopping zabbix') self.run = False self.event.set() def serve(self) -> None: self.log.info('Zabbix module starting up') self.run = True self.init_module_config() discovery_interval = self.config['discovery_interval'] # We are sending discovery once plugin is loaded discovery_counter = cast(int, discovery_interval) while self.run: self.log.debug('Waking up for new iteration') if discovery_counter == discovery_interval: try: self.discovery() except Exception: # Shouldn't happen, but let's log it and retry next interval, # rather than dying completely. self.log.exception("Unexpected error during discovery():") finally: discovery_counter = 0 try: data = self.get_data() self.send(data) except Exception: # Shouldn't happen, but let's log it and retry next interval, # rather than dying completely. self.log.exception("Unexpected error during send():") interval = cast(float, self.config['interval']) self.log.debug('Sleeping for %d seconds', interval) discovery_counter += 1 self.event.wait(interval) def self_test(self) -> None: data = self.get_data() if data['overall_status'] not in self.ceph_health_mapping: raise RuntimeError('No valid overall_status found in data') int(data['overall_status_int']) if data['num_mon'] < 1: raise RuntimeError('num_mon is smaller than 1')
class Module(MgrModule): COMMANDS = [{ "cmd": "progress", "desc": "Show progress of recovery operations", "perm": "r" }, { "cmd": "progress json", "desc": "Show machine readable progress information", "perm": "r" }, { "cmd": "progress clear", "desc": "Reset progress tracking", "perm": "rw" }, { "cmd": "progress on", "desc": "Enable progress tracking", "perm": "rw" }, { "cmd": "progress off", "desc": "Disable progress tracking", "perm": "rw" }] MODULE_OPTIONS = [ Option('max_completed_events', default=50, type='int', desc='number of past completed events to remember', runtime=True), Option('persist_interval', default=5, type='secs', desc='how frequently to persist completed events', runtime=True), Option( 'enabled', default=True, type='bool', ) ] def __init__(self, *args, **kwargs): super(Module, self).__init__(*args, **kwargs) self._events = { } # type: Dict[str, Union[RemoteEvent, PgRecoveryEvent, GlobalRecoveryEvent]] self._completed_events = [] # type: List[GhostEvent] self._old_osd_map = None # type: Optional[OSDMap] self._ready = threading.Event() self._shutdown = threading.Event() self._latest_osdmap = None # type: Optional[OSDMap] self._dirty = False global _module _module = self # only for mypy if TYPE_CHECKING: self.max_completed_events = 0 self.persist_interval = 0 self.enabled = True def config_notify(self): for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name'])) def _osd_in_out(self, old_map, old_dump, new_map, osd_id, marked): # type: (OSDMap, Dict, OSDMap, str, str) -> None # A function that will create or complete an event when an # OSD is marked in or out according to the affected PGs affected_pgs = [] unmoved_pgs = [] for pool in old_dump['pools']: pool_id = pool['pool'] # type: str for ps in range(0, pool['pg_num']): # Was this OSD affected by the OSD coming in/out? # Compare old and new osds using # data from the json dump old_up_acting = old_map.pg_to_up_acting_osds(pool['pool'], ps) old_osds = set(old_up_acting['acting']) new_up_acting = new_map.pg_to_up_acting_osds(pool['pool'], ps) new_osds = set(new_up_acting['acting']) # Check the osd_id being in the acting set for both old # and new maps to cover both out and in cases was_on_out_or_in_osd = osd_id in old_osds or osd_id in new_osds if not was_on_out_or_in_osd: continue self.log.debug("pool_id, ps = {0}, {1}".format(pool_id, ps)) self.log.debug("old_up_acting: {0}".format( json.dumps(old_up_acting, indent=4, sort_keys=True))) # Has this OSD been assigned a new location? # (it might not be if there is no suitable place to move # after an OSD is marked in/out) if marked == "in": is_relocated = len(old_osds - new_osds) > 0 else: is_relocated = len(new_osds - old_osds) > 0 self.log.debug("new_up_acting: {0}".format( json.dumps(new_up_acting, indent=4, sort_keys=True))) if was_on_out_or_in_osd and is_relocated: # This PG is now in motion, track its progress affected_pgs.append(PgId(pool_id, ps)) elif not is_relocated: # This PG didn't get a new location, we'll log it unmoved_pgs.append(PgId(pool_id, ps)) # In the case that we ignored some PGs, log the reason why (we may # not end up creating a progress event) if len(unmoved_pgs): self.log.warning( "{0} PGs were on osd.{1}, but didn't get new locations".format( len(unmoved_pgs), osd_id)) self.log.warning("{0} PGs affected by osd.{1} being marked {2}".format( len(affected_pgs), osd_id, marked)) # In the case of the osd coming back in, we might need to cancel # previous recovery event for that osd if marked == "in": for ev_id in list(self._events): ev = self._events[ev_id] if isinstance(ev, PgRecoveryEvent) and osd_id in ev.which_osds: self.log.info( "osd.{0} came back in, cancelling event".format( osd_id)) self._complete(ev) if len(affected_pgs) > 0: r_ev = PgRecoveryEvent( "Rebalancing after osd.{0} marked {1}".format(osd_id, marked), refs=[("osd", osd_id)], which_pgs=affected_pgs, which_osds=[osd_id], start_epoch=self.get_osdmap().get_epoch(), add_to_ceph_s=False) r_ev.pg_update(self.get("pg_stats"), self.get("pg_ready"), self.log) self._events[r_ev.id] = r_ev def _osdmap_changed(self, old_osdmap, new_osdmap): # type: (OSDMap, OSDMap) -> None old_dump = old_osdmap.dump() new_dump = new_osdmap.dump() old_osds = dict([(o['osd'], o) for o in old_dump['osds']]) for osd in new_dump['osds']: osd_id = osd['osd'] new_weight = osd['in'] if osd_id in old_osds: old_weight = old_osds[osd_id]['in'] if new_weight == 0.0 and old_weight > new_weight: self.log.warning("osd.{0} marked out".format(osd_id)) self._osd_in_out(old_osdmap, old_dump, new_osdmap, osd_id, "out") elif new_weight >= 1.0 and old_weight == 0.0: # Only consider weight>=1.0 as "in" to avoid spawning # individual recovery events on every adjustment # in a gradual weight-in self.log.warning("osd.{0} marked in".format(osd_id)) self._osd_in_out(old_osdmap, old_dump, new_osdmap, osd_id, "in") def _pg_state_changed(self, pg_dump): # This function both constructs and updates # the global recovery event if one of the # PGs is not at active+clean state pgs = pg_dump['pg_stats'] total_pg_num = len(pgs) active_clean_num = 0 for pg in pgs: state = pg['state'] states = state.split("+") if "active" in states and "clean" in states: active_clean_num += 1 try: # There might be a case where there is no pg_num progress = float(active_clean_num) / total_pg_num except ZeroDivisionError: return if progress < 1.0: ev = GlobalRecoveryEvent("Global Recovery Event", refs=[("global", "")], add_to_ceph_s=True, start_epoch=self.get_osdmap().get_epoch(), active_clean_num=active_clean_num) ev.global_event_update_progress(pg_dump) self._events[ev.id] = ev def notify(self, notify_type, notify_data): self._ready.wait() if not self.enabled: return if notify_type == "osd_map": old_osdmap = self._latest_osdmap self._latest_osdmap = self.get_osdmap() assert old_osdmap assert self._latest_osdmap self.log.info("Processing OSDMap change {0}..{1}".format( old_osdmap.get_epoch(), self._latest_osdmap.get_epoch())) self._osdmap_changed(old_osdmap, self._latest_osdmap) elif notify_type == "pg_summary": # if there are no events we will skip this here to avoid # expensive get calls if len(self._events) == 0: return global_event = False data = self.get("pg_stats") ready = self.get("pg_ready") for ev_id in list(self._events): ev = self._events[ev_id] # Check for types of events # we have to update if isinstance(ev, PgRecoveryEvent): ev.pg_update(data, ready, self.log) self.maybe_complete(ev) elif isinstance(ev, GlobalRecoveryEvent): global_event = True ev.global_event_update_progress(data) self.maybe_complete(ev) if not global_event: # If there is no global event # we create one self._pg_state_changed(data) def maybe_complete(self, event): # type: (Event) -> None if event.progress >= 1.0: self._complete(event) def _save(self): self.log.info("Writing back {0} completed events".format( len(self._completed_events))) # TODO: bound the number we store. encoded = json.dumps({ "events": [ev.to_json() for ev in self._completed_events], "version": ENCODING_VERSION, "compat_version": ENCODING_VERSION }) self.set_store("completed", encoded) def _load(self): stored = self.get_store("completed") if stored is None: self.log.info("No stored events to load") return decoded = json.loads(stored) if decoded['compat_version'] > ENCODING_VERSION: raise RuntimeError("Cannot decode version {0}".format( decoded['compat_version'])) if decoded['compat_version'] < ENCODING_VERSION: # we need to add the "started_at" and "finished_at" attributes to the events for ev in decoded['events']: ev['started_at'] = None ev['finished_at'] = None for ev in decoded['events']: self._completed_events.append( GhostEvent(ev['id'], ev['message'], ev['refs'], ev['started_at'], ev['finished_at'], ev.get('failed', False), ev.get('failure_message'))) self._prune_completed_events() def _prune_completed_events(self): length = len(self._completed_events) if length > self.max_completed_events: self._completed_events = self._completed_events[ length - self.max_completed_events:length] self._dirty = True def serve(self): self.config_notify() self.clear_all_progress_events() self.log.info("Loading...") self._load() self.log.info("Loaded {0} historic events".format( self._completed_events)) self._latest_osdmap = self.get_osdmap() self.log.info("Loaded OSDMap, ready.") self._ready.set() while not self._shutdown.is_set(): # Lazy periodic write back of completed events if self._dirty: self._save() self._dirty = False self._shutdown.wait(timeout=self.persist_interval) self._shutdown.wait() def shutdown(self): self._shutdown.set() self.clear_all_progress_events() def update(self, ev_id, ev_msg, ev_progress, refs=None, add_to_ceph_s=False): # type: (str, str, float, Optional[list], bool) -> None """ For calling from other mgr modules """ if not self.enabled: return if refs is None: refs = [] try: ev = self._events[ev_id] assert isinstance(ev, RemoteEvent) except KeyError: ev = RemoteEvent(ev_id, ev_msg, refs, add_to_ceph_s) self._events[ev_id] = ev self.log.info("update: starting ev {0} ({1})".format( ev_id, ev_msg)) else: self.log.debug("update: {0} on {1}".format(ev_progress, ev_msg)) ev.set_progress(ev_progress) ev.set_message(ev_msg) def _complete(self, ev): # type: (Event) -> None duration = (time.time() - ev.started_at) self.log.info("Completed event {0} ({1}) in {2} seconds".format( ev.id, ev.message, int(round(duration)))) self.complete_progress_event(ev.id) self._completed_events.append( GhostEvent(ev.id, ev.message, ev.refs, ev.add_to_ceph_s, ev.started_at, failed=ev.failed, failure_message=ev.failure_message)) assert ev.id del self._events[ev.id] self._prune_completed_events() self._dirty = True def complete(self, ev_id): """ For calling from other mgr modules """ if not self.enabled: return try: ev = self._events[ev_id] assert isinstance(ev, RemoteEvent) ev.set_progress(1.0) self.log.info("complete: finished ev {0} ({1})".format( ev_id, ev.message)) self._complete(ev) except KeyError: self.log.warning("complete: ev {0} does not exist".format(ev_id)) pass def fail(self, ev_id, message): """ For calling from other mgr modules to mark an event as failed (and complete) """ try: ev = self._events[ev_id] assert isinstance(ev, RemoteEvent) ev.set_failed(message) self.log.info("fail: finished ev {0} ({1}): {2}".format( ev_id, ev.message, message)) self._complete(ev) except KeyError: self.log.warning("fail: ev {0} does not exist".format(ev_id)) def on(self): self.set_module_option('enabled', "true") def off(self): self.set_module_option('enabled', "false") def _handle_ls(self): if len(self._events) or len(self._completed_events): out = "" chrono_order = sorted(self._events.values(), key=lambda x: x.started_at, reverse=True) for ev in chrono_order: out += ev.twoline_progress() out += "\n" if len(self._completed_events): # TODO: limit number of completed events to show out += "\n" for ghost_ev in self._completed_events: out += "[{0}]: {1}\n".format( "Complete" if not ghost_ev.failed else "Failed", ghost_ev.twoline_progress()) return 0, out, "" else: return 0, "", "Nothing in progress" def _json(self): return { 'events': [ev.to_json() for ev in self._events.values()], 'completed': [ev.to_json() for ev in self._completed_events] } def clear(self): self._events = {} self._completed_events = [] self._dirty = True self._save() self.clear_all_progress_events() def _handle_clear(self): self.clear() return 0, "", "" def handle_command(self, _, cmd): if cmd['prefix'] == "progress": return self._handle_ls() elif cmd['prefix'] == "progress clear": # The clear command isn't usually needed - it's to enable # the admin to "kick" this module if it seems to have done # something wrong (e.g. we have a bug causing a progress event # that never finishes) return self._handle_clear() elif cmd['prefix'] == "progress json": return 0, json.dumps(self._json(), indent=4, sort_keys=True), "" elif cmd['prefix'] == "progress on": if self.enabled: return 0, "", "progress already enabled!" self.on() return 0, "", "progress enabled" elif cmd['prefix'] == "progress off": if not self.enabled: return 0, "", "progress already disabled!" self.off() self.clear() return 0, "", "progress disabled" else: raise NotImplementedError(cmd['prefix'])
class RookOrchestrator(MgrModule, orchestrator.Orchestrator): """ Writes are a two-phase thing, firstly sending the write to the k8s API (fast) and then waiting for the corresponding change to appear in the Ceph cluster (slow) Right now, we are calling the k8s API synchronously. """ MODULE_OPTIONS: List[Option] = [ # TODO: configure k8s API addr instead of assuming local Option( 'storage_class', type='str', default='local', desc='storage class name for LSO-discovered PVs', ), ] @staticmethod def can_run() -> Tuple[bool, str]: if not kubernetes_imported: return False, "`kubernetes` python module not found" if not RookEnv().api_version_match(): return False, "Rook version unsupported." return True, '' def available(self) -> Tuple[bool, str, Dict[str, Any]]: if not kubernetes_imported: return False, "`kubernetes` python module not found", {} elif not self._rook_env.has_namespace(): return False, "ceph-mgr not running in Rook cluster", {} try: self.k8s.list_namespaced_pod(self._rook_env.namespace) except ApiException as e: return False, "Cannot reach Kubernetes API: {}".format(e), {} else: return True, "", {} def __init__(self, *args: Any, **kwargs: Any) -> None: super(RookOrchestrator, self).__init__(*args, **kwargs) self._initialized = threading.Event() self._k8s_CoreV1_api: Optional[client.CoreV1Api] = None self._k8s_BatchV1_api: Optional[client.BatchV1Api] = None self._k8s_CustomObjects_api: Optional[client.CustomObjectsApi] = None self._k8s_StorageV1_api: Optional[client.StorageV1Api] = None self._rook_cluster: Optional[RookCluster] = None self._rook_env = RookEnv() self.storage_class = self.get_module_option('storage_class') self._shutdown = threading.Event() def config_notify(self) -> None: """ This method is called whenever one of our config options is changed. TODO: this method should be moved into mgr_module.py """ for opt in self.MODULE_OPTIONS: setattr( self, opt['name'], # type: ignore self.get_module_option(opt['name'])) # type: ignore self.log.debug(' mgr option %s = %s', opt['name'], getattr(self, opt['name'])) # type: ignore assert isinstance(self.storage_class, str) self.rook_cluster.storage_class = self.storage_class def shutdown(self) -> None: self._shutdown.set() @property def k8s(self): # type: () -> client.CoreV1Api self._initialized.wait() assert self._k8s_CoreV1_api is not None return self._k8s_CoreV1_api @property def rook_cluster(self): # type: () -> RookCluster self._initialized.wait() assert self._rook_cluster is not None return self._rook_cluster def serve(self) -> None: # For deployed clusters, we should always be running inside # a Rook cluster. For development convenience, also support # running outside (reading ~/.kube config) if self._rook_env.has_namespace(): config.load_incluster_config() else: self.log.warning("DEVELOPMENT ONLY: Reading kube config from ~") config.load_kube_config() # So that I can do port forwarding from my workstation - jcsp from kubernetes.client import configuration configuration.verify_ssl = False self._k8s_CoreV1_api = client.CoreV1Api() self._k8s_BatchV1_api = client.BatchV1Api() self._k8s_CustomObjects_api = client.CustomObjectsApi() self._k8s_StorageV1_api = client.StorageV1Api() try: # XXX mystery hack -- I need to do an API call from # this context, or subsequent API usage from handle_command # fails with SSLError('bad handshake'). Suspect some kind of # thread context setup in SSL lib? self._k8s_CoreV1_api.list_namespaced_pod(self._rook_env.namespace) except ApiException: # Ignore here to make self.available() fail with a proper error message pass assert isinstance(self.storage_class, str) self._rook_cluster = RookCluster(self._k8s_CoreV1_api, self._k8s_BatchV1_api, self._k8s_CustomObjects_api, self._k8s_StorageV1_api, self._rook_env, self.storage_class) self._initialized.set() while not self._shutdown.is_set(): self._shutdown.wait(5) @handle_orch_error def get_inventory( self, host_filter: Optional[orchestrator.InventoryFilter] = None, refresh: bool = False) -> List[orchestrator.InventoryHost]: host_list = None if host_filter and host_filter.hosts: # Explicit host list host_list = host_filter.hosts elif host_filter and host_filter.labels: # TODO: query k8s API to resolve to host list, and pass # it into RookCluster.get_discovered_devices raise NotImplementedError() discovered_devs = self.rook_cluster.get_discovered_devices(host_list) result = [] for host_name, host_devs in discovered_devs.items(): devs = [] for d in host_devs: devs.append(d) result.append( orchestrator.InventoryHost(host_name, inventory.Devices(devs))) return result @handle_orch_error def get_hosts(self): # type: () -> List[orchestrator.HostSpec] return [ orchestrator.HostSpec(n) for n in self.rook_cluster.get_node_names() ] @handle_orch_error def describe_service( self, service_type: Optional[str] = None, service_name: Optional[str] = None, refresh: bool = False) -> List[orchestrator.ServiceDescription]: now = datetime_now() # CephCluster cl = self.rook_cluster.rook_api_get("cephclusters/{0}".format( self.rook_cluster.rook_env.cluster_name)) self.log.debug('CephCluster %s' % cl) image_name = cl['spec'].get('cephVersion', {}).get('image', None) num_nodes = len(self.rook_cluster.get_node_names()) spec = {} if service_type == 'mon' or service_type is None: spec['mon'] = orchestrator.ServiceDescription( spec=ServiceSpec( 'mon', placement=PlacementSpec(count=cl['spec'].get( 'mon', {}).get('count', 1), ), ), size=cl['spec'].get('mon', {}).get('count', 1), container_image_name=image_name, last_refresh=now, ) if service_type == 'mgr' or service_type is None: spec['mgr'] = orchestrator.ServiceDescription( spec=ServiceSpec( 'mgr', placement=PlacementSpec.from_string('count:1'), ), size=1, container_image_name=image_name, last_refresh=now, ) if not cl['spec'].get('crashCollector', {}).get('disable', False): spec['crash'] = orchestrator.ServiceDescription( spec=ServiceSpec( 'crash', placement=PlacementSpec.from_string('*'), ), size=num_nodes, container_image_name=image_name, last_refresh=now, ) if service_type == 'mds' or service_type is None: # CephFilesystems all_fs = self.rook_cluster.rook_api_get("cephfilesystems/") self.log.debug('CephFilesystems %s' % all_fs) for fs in all_fs.get('items', []): svc = 'mds.' + fs['metadata']['name'] if svc in spec: continue # FIXME: we are conflating active (+ standby) with count active = fs['spec'].get('metadataServer', {}).get('activeCount', 1) total_mds = active if fs['spec'].get('metadataServer', {}).get('activeStandby', False): total_mds = active * 2 spec[svc] = orchestrator.ServiceDescription( spec=ServiceSpec( service_type='mds', service_id=fs['metadata']['name'], placement=PlacementSpec(count=active), ), size=total_mds, container_image_name=image_name, last_refresh=now, ) if service_type == 'rgw' or service_type is None: # CephObjectstores all_zones = self.rook_cluster.rook_api_get("cephobjectstores/") self.log.debug('CephObjectstores %s' % all_zones) for zone in all_zones.get('items', []): rgw_realm = zone['metadata']['name'] rgw_zone = rgw_realm svc = 'rgw.' + rgw_realm + '.' + rgw_zone if svc in spec: continue active = zone['spec']['gateway']['instances'] if 'securePort' in zone['spec']['gateway']: ssl = True port = zone['spec']['gateway']['securePort'] else: ssl = False port = zone['spec']['gateway']['port'] or 80 spec[svc] = orchestrator.ServiceDescription( spec=RGWSpec( service_id=rgw_realm + '.' + rgw_zone, rgw_realm=rgw_realm, rgw_zone=rgw_zone, ssl=ssl, rgw_frontend_port=port, placement=PlacementSpec(count=active), ), size=active, container_image_name=image_name, last_refresh=now, ) if service_type == 'nfs' or service_type is None: # CephNFSes all_nfs = self.rook_cluster.rook_api_get("cephnfses/") self.log.warning('CephNFS %s' % all_nfs) for nfs in all_nfs.get('items', []): nfs_name = nfs['metadata']['name'] svc = 'nfs.' + nfs_name if svc in spec: continue active = nfs['spec'].get('server', {}).get('active') spec[svc] = orchestrator.ServiceDescription( spec=NFSServiceSpec( service_id=nfs_name, placement=PlacementSpec(count=active), ), size=active, last_refresh=now, ) for dd in self._list_daemons(): if dd.service_name() not in spec: continue service = spec[dd.service_name()] service.running += 1 if not service.container_image_id: service.container_image_id = dd.container_image_id if not service.container_image_name: service.container_image_name = dd.container_image_name if service.last_refresh is None or not dd.last_refresh or dd.last_refresh < service.last_refresh: service.last_refresh = dd.last_refresh if service.created is None or dd.created is None or dd.created < service.created: service.created = dd.created return [v for k, v in spec.items()] @handle_orch_error def list_daemons( self, service_name: Optional[str] = None, daemon_type: Optional[str] = None, daemon_id: Optional[str] = None, host: Optional[str] = None, refresh: bool = False) -> List[orchestrator.DaemonDescription]: return self._list_daemons(service_name=service_name, daemon_type=daemon_type, daemon_id=daemon_id, host=host, refresh=refresh) def _list_daemons( self, service_name: Optional[str] = None, daemon_type: Optional[str] = None, daemon_id: Optional[str] = None, host: Optional[str] = None, refresh: bool = False) -> List[orchestrator.DaemonDescription]: pods = self.rook_cluster.describe_pods(daemon_type, daemon_id, host) self.log.debug('pods %s' % pods) result = [] for p in pods: sd = orchestrator.DaemonDescription() sd.hostname = p['hostname'] sd.daemon_type = p['labels']['app'].replace('rook-ceph-', '') status = { 'Pending': orchestrator.DaemonDescriptionStatus.error, 'Running': orchestrator.DaemonDescriptionStatus.running, 'Succeeded': orchestrator.DaemonDescriptionStatus.stopped, 'Failed': orchestrator.DaemonDescriptionStatus.error, 'Unknown': orchestrator.DaemonDescriptionStatus.error, }[p['phase']] sd.status = status sd.status_desc = p['phase'] if 'ceph_daemon_id' in p['labels']: sd.daemon_id = p['labels']['ceph_daemon_id'] elif 'ceph-osd-id' in p['labels']: sd.daemon_id = p['labels']['ceph-osd-id'] else: # Unknown type -- skip it continue if service_name is not None and service_name != sd.service_name(): continue sd.container_image_name = p['container_image_name'] sd.container_image_id = p['container_image_id'] sd.created = p['created'] sd.last_configured = p['created'] sd.last_deployed = p['created'] sd.started = p['started'] sd.last_refresh = p['refreshed'] result.append(sd) return result @handle_orch_error def remove_service(self, service_name: str) -> str: service_type, service_name = service_name.split('.', 1) if service_type == 'mds': return self.rook_cluster.rm_service('cephfilesystems', service_name) elif service_type == 'rgw': return self.rook_cluster.rm_service('cephobjectstores', service_name) elif service_type == 'nfs': return self.rook_cluster.rm_service('cephnfses', service_name) else: raise orchestrator.OrchestratorError( f'Service type {service_type} not supported') @handle_orch_error def apply_mon(self, spec): # type: (ServiceSpec) -> str if spec.placement.hosts or spec.placement.label: raise RuntimeError("Host list or label is not supported by rook.") return self.rook_cluster.update_mon_count(spec.placement.count) @handle_orch_error def apply_mds(self, spec): # type: (ServiceSpec) -> str return self.rook_cluster.apply_filesystem(spec) @handle_orch_error def apply_rgw(self, spec): # type: (RGWSpec) -> str return self.rook_cluster.apply_objectstore(spec) @handle_orch_error def apply_nfs(self, spec): # type: (NFSServiceSpec) -> str return self.rook_cluster.apply_nfsgw(spec) @handle_orch_error def remove_daemons(self, names: List[str]) -> List[str]: return self.rook_cluster.remove_pods(names) def apply_drivegroups( self, specs: List[DriveGroupSpec]) -> OrchResult[List[str]]: result_list = [] all_hosts = raise_if_exception(self.get_hosts()) for drive_group in specs: matching_hosts = drive_group.placement.filter_matching_hosts( lambda label=None, as_hostspec=None: all_hosts) if not self.rook_cluster.node_exists(matching_hosts[0]): raise RuntimeError("Node '{0}' is not in the Kubernetes " "cluster".format(matching_hosts)) # Validate whether cluster CRD can accept individual OSD # creations (i.e. not useAllDevices) if not self.rook_cluster.can_create_osd(): raise RuntimeError("Rook cluster configuration does not " "support OSD creation.") result_list.append( self.rook_cluster.add_osds(drive_group, matching_hosts)) return OrchResult(result_list) """ @handle_orch_error def create_osds(self, drive_group): # type: (DriveGroupSpec) -> str # Creates OSDs from a drive group specification. # $: ceph orch osd create -i <dg.file> # The drivegroup file must only contain one spec at a time. # targets = [] # type: List[str] if drive_group.data_devices and drive_group.data_devices.paths: targets += [d.path for d in drive_group.data_devices.paths] if drive_group.data_directories: targets += drive_group.data_directories all_hosts = raise_if_exception(self.get_hosts()) matching_hosts = drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: all_hosts) assert len(matching_hosts) == 1 if not self.rook_cluster.node_exists(matching_hosts[0]): raise RuntimeError("Node '{0}' is not in the Kubernetes " "cluster".format(matching_hosts)) # Validate whether cluster CRD can accept individual OSD # creations (i.e. not useAllDevices) if not self.rook_cluster.can_create_osd(): raise RuntimeError("Rook cluster configuration does not " "support OSD creation.") return self.rook_cluster.add_osds(drive_group, matching_hosts) # TODO: this was the code to update the progress reference: @handle_orch_error def has_osds(matching_hosts: List[str]) -> bool: # Find OSD pods on this host pod_osd_ids = set() pods = self.k8s.list_namespaced_pod(self._rook_env.namespace, label_selector="rook_cluster={},app=rook-ceph-osd".format(self._rook_env.cluster_name), field_selector="spec.nodeName={0}".format( matching_hosts[0] )).items for p in pods: pod_osd_ids.add(int(p.metadata.labels['ceph-osd-id'])) self.log.debug('pod_osd_ids={0}'.format(pod_osd_ids)) found = [] osdmap = self.get("osd_map") for osd in osdmap['osds']: osd_id = osd['osd'] if osd_id not in pod_osd_ids: continue metadata = self.get_metadata('osd', "%s" % osd_id) if metadata and metadata['devices'] in targets: found.append(osd_id) else: self.log.info("ignoring osd {0} {1}".format( osd_id, metadata['devices'] if metadata else 'DNE' )) return found is not None """ @handle_orch_error def blink_device_light( self, ident_fault: str, on: bool, locs: List[orchestrator.DeviceLightLoc]) -> List[str]: return self.rook_cluster.blink_light(ident_fault, on, locs)
class Module(MgrModule, CherryPyConfig): """ dashboard module entrypoint """ COMMANDS = [ { 'cmd': 'dashboard set-jwt-token-ttl ' 'name=seconds,type=CephInt', 'desc': 'Set the JWT token TTL in seconds', 'perm': 'w' }, { 'cmd': 'dashboard get-jwt-token-ttl', 'desc': 'Get the JWT token TTL in seconds', 'perm': 'r' }, { "cmd": "dashboard create-self-signed-cert", "desc": "Create self signed certificate", "perm": "w" }, { "cmd": "dashboard grafana dashboards update", "desc": "Push dashboards to Grafana", "perm": "w", }, ] COMMANDS.extend(options_command_list()) COMMANDS.extend(SSO_COMMANDS) PLUGIN_MANAGER.hook.register_commands() MODULE_OPTIONS = [ Option(name='server_addr', type='str', default=get_default_addr()), Option(name='server_port', type='int', default=8080), Option(name='ssl_server_port', type='int', default=8443), Option(name='jwt_token_ttl', type='int', default=28800), Option(name='password', type='str', default=''), Option(name='url_prefix', type='str', default=''), Option(name='username', type='str', default=''), Option(name='key_file', type='str', default=''), Option(name='crt_file', type='str', default=''), Option(name='ssl', type='bool', default=True), Option(name='standby_behaviour', type='str', default='redirect', enum_allowed=['redirect', 'error']), Option(name='standby_error_status_code', type='int', default=500, min=400, max=599) ] MODULE_OPTIONS.extend(options_schema_list()) for options in PLUGIN_MANAGER.hook.get_options() or []: MODULE_OPTIONS.extend(options) __pool_stats = collections.defaultdict(lambda: collections.defaultdict( lambda: collections.deque(maxlen=10))) # type: dict def __init__(self, *args, **kwargs): super(Module, self).__init__(*args, **kwargs) CherryPyConfig.__init__(self) mgr.init(self) self._stopping = threading.Event() self.shutdown_event = threading.Event() self.ACCESS_CTRL_DB = None self.SSO_DB = None @classmethod def can_run(cls): if cherrypy is None: return False, "Missing dependency: cherrypy" if not os.path.exists(cls.get_frontend_path()): return False, "Frontend assets not found: incomplete build?" return True, "" @classmethod def get_frontend_path(cls): current_dir = os.path.dirname(os.path.abspath(__file__)) return os.path.join(current_dir, 'frontend/dist') def serve(self): AuthManager.initialize() load_sso_db() uri = self.await_configuration() if uri is None: # We were shut down while waiting return # Publish the URI that others may use to access the service we're # about to start serving self.set_uri(uri) mapper, parent_urls = generate_routes(self.url_prefix) config = {} for purl in parent_urls: config[purl] = {'request.dispatch': mapper} cherrypy.tree.mount(None, config=config) PLUGIN_MANAGER.hook.setup() cherrypy.engine.start() NotificationQueue.start_queue() TaskManager.init() logger.info('Engine started.') update_dashboards = str_to_bool( self.get_module_option('GRAFANA_UPDATE_DASHBOARDS', 'False')) if update_dashboards: logger.info('Starting Grafana dashboard task') TaskManager.run( 'grafana/dashboards/update', {}, push_local_dashboards, kwargs=dict(tries=10, sleep=60), ) # wait for the shutdown event self.shutdown_event.wait() self.shutdown_event.clear() NotificationQueue.stop() cherrypy.engine.stop() logger.info('Engine stopped') def shutdown(self): super(Module, self).shutdown() CherryPyConfig.shutdown(self) logger.info('Stopping engine...') self.shutdown_event.set() @CLIWriteCommand("dashboard set-ssl-certificate", "name=mgr_id,type=CephString,req=false") def set_ssl_certificate(self, mgr_id=None, inbuf=None): if inbuf is None: return -errno.EINVAL, '',\ 'Please specify the certificate file with "-i" option' if mgr_id is not None: self.set_store('{}/crt'.format(mgr_id), inbuf) else: self.set_store('crt', inbuf) return 0, 'SSL certificate updated', '' @CLIWriteCommand("dashboard set-ssl-certificate-key", "name=mgr_id,type=CephString,req=false") def set_ssl_certificate_key(self, mgr_id=None, inbuf=None): if inbuf is None: return -errno.EINVAL, '',\ 'Please specify the certificate key file with "-i" option' if mgr_id is not None: self.set_store('{}/key'.format(mgr_id), inbuf) else: self.set_store('key', inbuf) return 0, 'SSL certificate key updated', '' def handle_command(self, inbuf, cmd): # pylint: disable=too-many-return-statements res = handle_option_command(cmd) if res[0] != -errno.ENOSYS: return res res = handle_sso_command(cmd) if res[0] != -errno.ENOSYS: return res if cmd['prefix'] == 'dashboard set-jwt-token-ttl': self.set_module_option('jwt_token_ttl', str(cmd['seconds'])) return 0, 'JWT token TTL updated', '' if cmd['prefix'] == 'dashboard get-jwt-token-ttl': ttl = self.get_module_option('jwt_token_ttl', JwtManager.JWT_TOKEN_TTL) return 0, str(ttl), '' if cmd['prefix'] == 'dashboard create-self-signed-cert': self.create_self_signed_cert() return 0, 'Self-signed certificate created', '' if cmd['prefix'] == 'dashboard grafana dashboards update': push_local_dashboards() return 0, 'Grafana dashboards updated', '' return (-errno.EINVAL, '', 'Command not found \'{0}\''.format(cmd['prefix'])) def create_self_signed_cert(self): cert, pkey = create_self_signed_cert('IT', 'ceph-dashboard') self.set_store('crt', cert) self.set_store('key', pkey) def notify(self, notify_type, notify_id): NotificationQueue.new_notification(notify_type, notify_id) def get_updated_pool_stats(self): df = self.get('df') pool_stats = {p['id']: p['stats'] for p in df['pools']} now = time.time() for pool_id, stats in pool_stats.items(): for stat_name, stat_val in stats.items(): self.__pool_stats[pool_id][stat_name].append((now, stat_val)) return self.__pool_stats
class Module(MgrModule): """ This module is for testing the ceph-mgr python interface from within a running ceph-mgr daemon. It implements a sychronous self-test command for calling the functions in the MgrModule interface one by one, and a background "workload" command for causing the module to perform some thrashing-type activities in its serve() thread. """ # The test code in qa/ relies on these options existing -- they # are of course not really used for anything in the module MODULE_OPTIONS = [ Option(name='testkey'), Option(name='testlkey'), Option(name='testnewline'), Option(name='roption1'), Option(name='roption2', type='str', default='xyz'), Option(name='rwoption1'), Option(name='rwoption2', type='int'), Option(name='rwoption3', type='float'), Option(name='rwoption4', type='str'), Option(name='rwoption5', type='bool'), Option(name='rwoption6', type='bool', default=True), Option(name='rwoption7', type='int', min=1, max=42), ] def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) self._event = threading.Event() self._workload: Optional[Workload] = None self._health: Dict[str, Dict[str, Any]] = {} self._repl = InteractiveInterpreter(dict(mgr=self)) @CLICommand('mgr self-test python-version', perm='r') def python_version(self) -> Tuple[int, str, str]: ''' Query the version of the embedded Python runtime ''' major = sys.version_info.major minor = sys.version_info.minor micro = sys.version_info.micro return 0, f'{major}.{minor}.{micro}', '' @CLICommand('mgr self-test run') def run(self) -> Tuple[int, str, str]: ''' Run mgr python interface tests ''' self._self_test() return 0, '', 'Self-test succeeded' @CLICommand('mgr self-test background start') def backgroun_start(self, workload: Workload) -> Tuple[int, str, str]: ''' Activate a background workload (one of command_spam, throw_exception) ''' self._workload = workload self._event.set() return 0, '', 'Running `{0}` in background'.format(self._workload) @CLICommand('mgr self-test background stop') def background_stop(self) -> Tuple[int, str, str]: ''' Stop background workload if any is running ''' if self._workload: was_running = self._workload self._workload = None self._event.set() return 0, '', 'Stopping background workload `{0}`'.format( was_running) else: return 0, '', 'No background workload was running' @CLICommand('mgr self-test config get') def config_get(self, key: str) -> Tuple[int, str, str]: ''' Peek at a configuration value ''' return 0, str(self.get_module_option(key)), '' @CLICommand('mgr self-test config get_localized') def config_get_localized(self, key: str) -> Tuple[int, str, str]: ''' Peek at a configuration value (localized variant) ''' return 0, str(self.get_localized_module_option(key)), '' @CLICommand('mgr self-test remote') def test_remote(self) -> Tuple[int, str, str]: ''' Test inter-module calls ''' self._test_remote_calls() return 0, '', 'Successfully called' @CLICommand('mgr self-test module') def module(self, module: str) -> Tuple[int, str, str]: ''' Run another module's self_test() method ''' try: r = self.remote(module, "self_test") except RuntimeError as e: return -1, '', "Test failed: {0}".format(e) else: return 0, str(r), "Self-test OK" @CLICommand('mgr self-test cluster-log') def do_cluster_log(self, channel: str, priority: str, message: str) -> Tuple[int, str, str]: ''' Create an audit log record. ''' priority_map = { 'info': self.ClusterLogPrio.INFO, 'security': self.ClusterLogPrio.SEC, 'warning': self.ClusterLogPrio.WARN, 'error': self.ClusterLogPrio.ERROR } self.cluster_log(channel, priority_map[priority], message) return 0, '', 'Successfully called' @CLICommand('mgr self-test health set') def health_set(self, checks: str) -> Tuple[int, str, str]: ''' Set a health check from a JSON-formatted description. ''' try: health_check = json.loads(checks) except Exception as e: return -1, "", "Failed to decode JSON input: {}".format(e) try: for check, info in health_check.items(): self._health[check] = { "severity": str(info["severity"]), "summary": str(info["summary"]), "count": 123, "detail": [str(m) for m in info["detail"]] } except Exception as e: return -1, "", "Invalid health check format: {}".format(e) self.set_health_checks(self._health) return 0, "", "" @CLICommand('mgr self-test health clear') def health_clear(self, checks: Optional[List[str]] = None) -> Tuple[int, str, str]: ''' Clear health checks by name. If no names provided, clear all. ''' if checks is not None: for check in checks: if check in self._health: del self._health[check] else: self._health = dict() self.set_health_checks(self._health) return 0, "", "" @CLICommand('mgr self-test insights_set_now_offset') def insights_set_now_offset(self, hours: int) -> Tuple[int, str, str]: ''' Set the now time for the insights module. ''' self.remote("insights", "testing_set_now_time_offset", hours) return 0, "", "" def _self_test(self) -> None: self.log.info("Running self-test procedure...") self._self_test_osdmap() self._self_test_getters() self._self_test_config() self._self_test_store() self._self_test_misc() self._self_test_perf_counters() def _self_test_getters(self) -> None: self.version self.get_context() self.get_mgr_id() # In this function, we will assume that the system is in a steady # state, i.e. if a server/service appears in one call, it will # not have gone by the time we call another function referring to it objects = [ "fs_map", "osdmap_crush_map_text", "osd_map", "config", "mon_map", "service_map", "osd_metadata", "pg_summary", "pg_status", "pg_dump", "pg_ready", "df", "pg_stats", "pool_stats", "osd_stats", "osd_ping_times", "health", "mon_status", "mgr_map" ] for obj in objects: assert self.get(obj) is not None assert self.get("__OBJ_DNE__") is None servers = self.list_servers() for server in servers: self.get_server(server['hostname']) # type: ignore osdmap = self.get('osd_map') for o in osdmap['osds']: osd_id = o['osd'] self.get_metadata("osd", str(osd_id)) self.get_daemon_status("osd", "0") def _self_test_config(self) -> None: # This is not a strong test (can't tell if values really # persisted), it's just for the python interface bit. self.set_module_option("testkey", "testvalue") assert self.get_module_option("testkey") == "testvalue" self.set_localized_module_option("testkey", "foo") assert self.get_localized_module_option("testkey") == "foo" # Must return the default value defined in MODULE_OPTIONS. value = self.get_localized_module_option("rwoption6") assert isinstance(value, bool) assert value is True # Use default value. assert self.get_module_option("roption1") is None assert self.get_module_option("roption1", "foobar") == "foobar" assert self.get_module_option("roption2") == "xyz" assert self.get_module_option("roption2", "foobar") == "xyz" # Option type is not defined => return as string. self.set_module_option("rwoption1", 8080) value = self.get_module_option("rwoption1") assert isinstance(value, str) assert value == "8080" # Option type is defined => return as integer. self.set_module_option("rwoption2", 10) value = self.get_module_option("rwoption2") assert isinstance(value, int) assert value == 10 # Option type is defined => return as float. self.set_module_option("rwoption3", 1.5) value = self.get_module_option("rwoption3") assert isinstance(value, float) assert value == 1.5 # Option type is defined => return as string. self.set_module_option("rwoption4", "foo") value = self.get_module_option("rwoption4") assert isinstance(value, str) assert value == "foo" # Option type is defined => return as bool. self.set_module_option("rwoption5", False) value = self.get_module_option("rwoption5") assert isinstance(value, bool) assert value is False # Option value range is specified try: self.set_module_option("rwoption7", 43) except Exception as e: assert isinstance(e, ValueError) else: message = "should raise if value is not in specified range" assert False, message # Specified module does not exist => return None. assert self.get_module_option_ex("foo", "bar") is None # Specified key does not exist => return None. assert self.get_module_option_ex("dashboard", "bar") is None self.set_module_option_ex("telemetry", "contact", "*****@*****.**") assert self.get_module_option_ex("telemetry", "contact") == "*****@*****.**" # No option default value, so use the specified one. assert self.get_module_option_ex("dashboard", "password") is None assert self.get_module_option_ex("dashboard", "password", "foobar") == "foobar" # Option type is not defined => return as string. self.set_module_option_ex("selftest", "rwoption1", 1234) value = self.get_module_option_ex("selftest", "rwoption1") assert isinstance(value, str) assert value == "1234" # Option type is defined => return as integer. self.set_module_option_ex("telemetry", "interval", 60) value = self.get_module_option_ex("telemetry", "interval") assert isinstance(value, int) assert value == 60 # Option type is defined => return as bool. self.set_module_option_ex("telemetry", "leaderboard", True) value = self.get_module_option_ex("telemetry", "leaderboard") assert isinstance(value, bool) assert value is True def _self_test_store(self) -> None: existing_keys = set(self.get_store_prefix("test").keys()) self.set_store("testkey", "testvalue") assert self.get_store("testkey") == "testvalue" assert (set(self.get_store_prefix("test").keys()) == {"testkey"} | existing_keys) def _self_test_perf_counters(self) -> None: self.get_perf_schema("osd", "0") self.get_counter("osd", "0", "osd.op") # get_counter # get_all_perf_coutners def _self_test_misc(self) -> None: self.set_uri("http://this.is.a.test.com") self.set_health_checks({}) def _self_test_osdmap(self) -> None: osdmap = self.get_osdmap() osdmap.get_epoch() osdmap.get_crush_version() osdmap.dump() inc = osdmap.new_incremental() osdmap.apply_incremental(inc) inc.get_epoch() inc.dump() crush = osdmap.get_crush() crush.dump() crush.get_item_name(-1) crush.get_item_weight(-1) crush.find_takes() crush.get_take_weight_osd_map(-1) # osdmap.get_pools_by_take() # osdmap.calc_pg_upmaps() # osdmap.map_pools_pgs_up() # inc.set_osd_reweights # inc.set_crush_compat_weight_set_weights self.log.info("Finished self-test procedure.") def _test_remote_calls(self) -> None: # Test making valid call self.remote("influx", "self_test") # Test calling module that exists but isn't enabled # (arbitrarily pick a non-always-on module to use) disabled_module = "telegraf" mgr_map = self.get("mgr_map") assert disabled_module not in mgr_map['modules'] # (This works until the Z release in about 2027) latest_release = sorted(mgr_map['always_on_modules'].keys())[-1] assert disabled_module not in mgr_map['always_on_modules'][latest_release] try: self.remote(disabled_module, "handle_command", {"prefix": "influx self-test"}) except ImportError: pass else: raise RuntimeError("ImportError not raised for disabled module") # Test calling module that doesn't exist try: self.remote("idontexist", "self_test") except ImportError: pass else: raise RuntimeError("ImportError not raised for nonexistent module") # Test calling method that doesn't exist try: self.remote("influx", "idontexist") except NameError: pass else: raise RuntimeError("KeyError not raised") def remote_from_orchestrator_cli_self_test(self, what: str) -> Any: import orchestrator if what == 'OrchestratorError': return orchestrator.OrchResult(result=None, exception=orchestrator.OrchestratorError('hello, world')) elif what == "ZeroDivisionError": return orchestrator.OrchResult(result=None, exception=ZeroDivisionError('hello, world')) assert False, repr(what) def shutdown(self) -> None: self._workload = Workload.SHUTDOWN self._event.set() def _command_spam(self) -> None: self.log.info("Starting command_spam workload...") while not self._event.is_set(): osdmap = self.get_osdmap() dump = osdmap.dump() count = len(dump['osds']) i = int(random.random() * count) w = random.random() result = CommandResult('') self.send_command(result, 'mon', '', json.dumps({ 'prefix': 'osd reweight', 'id': i, 'weight': w}), '') _ = osdmap.get_crush().dump() r, outb, outs = result.wait() self._event.clear() self.log.info("Ended command_spam workload...") @CLICommand('mgr self-test eval') def eval(self, s: Optional[str] = None, inbuf: Optional[str] = None) -> HandleCommandResult: ''' eval given source ''' source = s or inbuf if source is None: return HandleCommandResult(-1, '', 'source is not specified') err = StringIO() out = StringIO() with redirect_stderr(err), redirect_stdout(out): needs_more = self._repl.runsource(source) if needs_more: retval = 2 stdout = '' stderr = '' else: retval = 0 stdout = out.getvalue() stderr = err.getvalue() return HandleCommandResult(retval, stdout, stderr) def serve(self) -> None: while True: if self._workload == Workload.COMMAND_SPAM: self._command_spam() elif self._workload == Workload.SHUTDOWN: self.log.info("Shutting down...") break elif self._workload == Workload.THROW_EXCEPTION: raise RuntimeError("Synthetic exception in serve") else: self.log.info("Waiting for workload request...") self._event.wait() self._event.clear()
class Module(MgrModule): MODULE_OPTIONS = [ Option(name='warn_recent_interval', type='secs', default=60 * 60 * 24 * 14, desc='time interval in which to warn about recent crashes', runtime=True), Option(name='retain_interval', type='secs', default=60 * 60 * 24 * 365, desc='how long to retain crashes before pruning them', runtime=True), ] def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) self.crashes: Optional[Dict[str, CrashT]] = None self.crashes_lock = Lock() self.run = True self.event = Event() if TYPE_CHECKING: self.warn_recent_interval = 0.0 self.retain_interval = 0.0 def shutdown(self) -> None: self.run = False self.event.set() def serve(self) -> None: self.config_notify() while self.run: with self.crashes_lock: self._refresh_health_checks() self._prune(self.retain_interval) wait = min(MAX_WAIT, max(self.warn_recent_interval / 100, MIN_WAIT)) self.event.wait(wait) self.event.clear() def config_notify(self) -> None: for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' mgr option %s = %s', opt['name'], getattr(self, opt['name'])) def _load_crashes(self) -> None: raw = self.get_store_prefix('crash/') self.crashes = {k[6:]: json.loads(m) for (k, m) in raw.items()} def _refresh_health_checks(self) -> None: if not self.crashes: self._load_crashes() assert self.crashes is not None cutoff = datetime.datetime.utcnow() - datetime.timedelta( seconds=self.warn_recent_interval) recent = { crashid: crash for crashid, crash in self.crashes.items() if (self.time_from_string(cast(str, crash['timestamp'])) > cutoff and 'archived' not in crash) } num = len(recent) health_checks: Dict[str, Dict[str, Union[int, str, List[str]]]] = {} if recent: detail = [ '%s crashed on host %s at %s' % (crash.get('entity_name', 'unidentified daemon'), crash.get('utsname_hostname', '(unknown)'), crash.get('timestamp', 'unknown time')) for crash in recent.values() ] if num > 30: detail = detail[0:30] detail.append('and %d more' % (num - 30)) self.log.debug('detail %s' % detail) health_checks['RECENT_CRASH'] = { 'severity': 'warning', 'summary': '%d daemons have recently crashed' % (num), 'count': num, 'detail': detail, } self.set_health_checks(health_checks) def time_from_string(self, timestr: str) -> datetime.datetime: # drop the 'Z' timezone indication, it's always UTC timestr = timestr.rstrip('Z') try: return datetime.datetime.strptime(timestr, DATEFMT) except ValueError: return datetime.datetime.strptime(timestr, OLD_DATEFMT) def validate_crash_metadata( self, inbuf: str) -> Dict[str, Union[str, List[str]]]: # raise any exceptions to caller metadata = json.loads(inbuf) for f in ['crash_id', 'timestamp']: if f not in metadata: raise AttributeError("missing '%s' field" % f) _ = self.time_from_string(metadata['timestamp']) return metadata def timestamp_filter( self, f: Callable[[datetime.datetime], bool]) -> Iterable[Tuple[str, CrashT]]: """ Filter crash reports by timestamp. :param f: f(time) return true to keep crash report :returns: crash reports for which f(time) returns true """ def inner(pair: Tuple[str, CrashT]) -> bool: _, crash = pair time = self.time_from_string(cast(str, crash["timestamp"])) return f(time) assert self.crashes is not None return filter(inner, self.crashes.items()) # stack signature helpers def sanitize_backtrace(self, bt: List[str]) -> List[str]: ret = list() for func_record in bt: # split into two fields on last space, take the first one, # strip off leading ( and trailing ) func_plus_offset = func_record.rsplit(' ', 1)[0][1:-1] ret.append(func_plus_offset.split('+')[0]) return ret ASSERT_MATCHEXPR = re.compile(r'(?s)(.*) thread .* time .*(: .*)\n') def sanitize_assert_msg(self, msg: str) -> str: # (?s) allows matching newline. get everything up to "thread" and # then after-and-including the last colon-space. This skips the # thread id, timestamp, and file:lineno, because file is already in # the beginning, and lineno may vary. matched = self.ASSERT_MATCHEXPR.match(msg) assert matched return ''.join(matched.groups()) def calc_sig(self, bt: List[str], assert_msg: Optional[str]) -> str: sig = hashlib.sha256() for func in self.sanitize_backtrace(bt): sig.update(func.encode()) if assert_msg: sig.update(self.sanitize_assert_msg(assert_msg).encode()) return ''.join('%02x' % c for c in sig.digest()) # command handlers @CLIReadCommand('crash info') @with_crashes def do_info(self, id: str) -> Tuple[int, str, str]: """ show crash dump metadata """ crashid = id assert self.crashes is not None crash = self.crashes.get(crashid) if not crash: return errno.EINVAL, '', 'crash info: %s not found' % crashid val = json.dumps(crash, indent=4, sort_keys=True) return 0, val, '' @CLICommand('crash post') def do_post(self, inbuf: str) -> Tuple[int, str, str]: """ Add a crash dump (use -i <jsonfile>) """ try: metadata = self.validate_crash_metadata(inbuf) except Exception as e: return errno.EINVAL, '', 'malformed crash metadata: %s' % e if 'backtrace' in metadata: backtrace = cast(List[str], metadata.get('backtrace')) assert_msg = cast(Optional[str], metadata.get('assert_msg')) metadata['stack_sig'] = self.calc_sig(backtrace, assert_msg) crashid = cast(str, metadata['crash_id']) assert self.crashes is not None if crashid not in self.crashes: self.crashes[crashid] = metadata key = 'crash/%s' % crashid self.set_store(key, json.dumps(metadata)) self._refresh_health_checks() return 0, '', '' def ls(self) -> Tuple[int, str, str]: if not self.crashes: self._load_crashes() return self.do_ls_all('') def _do_ls(self, t: Iterable[CrashT], format: Optional[str]) -> Tuple[int, str, str]: r = sorted(t, key=lambda i: i['crash_id']) if format in ('json', 'json-pretty'): return 0, json.dumps(r, indent=4, sort_keys=True), '' else: table = PrettyTable(['ID', 'ENTITY', 'NEW'], border=False) table.left_padding_width = 0 table.right_padding_width = 2 table.align['ID'] = 'l' table.align['ENTITY'] = 'l' for c in r: table.add_row([ c.get('crash_id'), c.get('entity_name', 'unknown'), '' if 'archived' in c else '*' ]) return 0, table.get_string(), '' @CLIReadCommand('crash ls') @with_crashes def do_ls_all(self, format: Optional[str] = None) -> Tuple[int, str, str]: """ Show new and archived crash dumps """ assert self.crashes is not None return self._do_ls(self.crashes.values(), format) @CLIReadCommand('crash ls-new') @with_crashes def do_ls_new(self, format: Optional[str] = None) -> Tuple[int, str, str]: """ Show new crash dumps """ assert self.crashes is not None t = [ crash for crashid, crash in self.crashes.items() if 'archived' not in crash ] return self._do_ls(t, format) @CLICommand('crash rm') @with_crashes def do_rm(self, id: str) -> Tuple[int, str, str]: """ Remove a saved crash <id> """ crashid = id assert self.crashes is not None if crashid in self.crashes: del self.crashes[crashid] key = 'crash/%s' % crashid self.set_store(key, None) # removes key self._refresh_health_checks() return 0, '', '' @CLICommand('crash prune') @with_crashes def do_prune(self, keep: int) -> Tuple[int, str, str]: """ Remove crashes older than <keep> days """ self._prune(keep * datetime.timedelta(days=1).total_seconds()) return 0, '', '' def _prune(self, seconds: float) -> None: now = datetime.datetime.utcnow() cutoff = now - datetime.timedelta(seconds=seconds) removed_any = False # make a copy of the list, since we'll modify self.crashes below to_prune = list(self.timestamp_filter(lambda ts: ts <= cutoff)) assert self.crashes is not None for crashid, crash in to_prune: del self.crashes[crashid] key = 'crash/%s' % crashid self.set_store(key, None) removed_any = True if removed_any: self._refresh_health_checks() @CLIWriteCommand('crash archive') @with_crashes def do_archive(self, id: str) -> Tuple[int, str, str]: """ Acknowledge a crash and silence health warning(s) """ crashid = id assert self.crashes is not None crash = self.crashes.get(crashid) if not crash: return errno.EINVAL, '', 'crash info: %s not found' % crashid if not crash.get('archived'): crash['archived'] = str(datetime.datetime.utcnow()) self.crashes[crashid] = crash key = 'crash/%s' % crashid self.set_store(key, json.dumps(crash)) self._refresh_health_checks() return 0, '', '' @CLIWriteCommand('crash archive-all') @with_crashes def do_archive_all(self) -> Tuple[int, str, str]: """ Acknowledge all new crashes and silence health warning(s) """ assert self.crashes is not None for crashid, crash in self.crashes.items(): if not crash.get('archived'): crash['archived'] = str(datetime.datetime.utcnow()) self.crashes[crashid] = crash key = 'crash/%s' % crashid self.set_store(key, json.dumps(crash)) self._refresh_health_checks() return 0, '', '' @CLIReadCommand('crash stat') @with_crashes def do_stat(self) -> Tuple[int, str, str]: """ Summarize recorded crashes """ # age in days for reporting, ordered smallest first AGE_IN_DAYS = [1, 3, 7] retlines = list() BinnedStatsT = Dict[str, Union[int, datetime.datetime, List[str]]] def binstr(bindict: BinnedStatsT) -> str: binlines = list() id_list = cast(List[str], bindict['idlist']) count = len(id_list) if count: binlines.append('%d older than %s days old:' % (count, bindict['age'])) for crashid in id_list: binlines.append(crashid) return '\n'.join(binlines) total = 0 now = datetime.datetime.utcnow() bins: List[BinnedStatsT] = [] for age in AGE_IN_DAYS: agelimit = now - datetime.timedelta(days=age) bins.append({'age': age, 'agelimit': agelimit, 'idlist': list()}) assert self.crashes is not None for crashid, crash in self.crashes.items(): total += 1 stamp = self.time_from_string(cast(str, crash['timestamp'])) for bindict in bins: if stamp <= cast(datetime.datetime, bindict['agelimit']): cast(List[str], bindict['idlist']).append(crashid) # don't count this one again continue retlines.append('%d crashes recorded' % total) for bindict in bins: retlines.append(binstr(bindict)) return 0, '\n'.join(retlines), '' @CLIReadCommand('crash json_report') @with_crashes def do_json_report(self, hours: int) -> Tuple[int, str, str]: """ Crashes in the last <hours> hours """ # Return a machine readable summary of recent crashes. report: DefaultDict[str, int] = defaultdict(lambda: 0) assert self.crashes is not None for crashid, crash in self.crashes.items(): pname = cast(str, crash.get("process_name", "unknown")) if not pname: pname = "unknown" report[pname] += 1 return 0, '', json.dumps(report, sort_keys=True) def self_test(self) -> None: # test time conversion timestr = '2018-06-22T20:35:38.058818Z' old_timestr = '2018-06-22 20:35:38.058818Z' dt = self.time_from_string(timestr) if dt != datetime.datetime(2018, 6, 22, 20, 35, 38, 58818): raise RuntimeError('time_from_string() failed') dt = self.time_from_string(old_timestr) if dt != datetime.datetime(2018, 6, 22, 20, 35, 38, 58818): raise RuntimeError('time_from_string() (old) failed')
class Alerts(MgrModule): MODULE_OPTIONS = [ Option(name='interval', type='secs', default=60, desc='How frequently to reexamine health status', runtime=True), # smtp Option(name='smtp_host', default='', desc='SMTP server', runtime=True), Option(name='smtp_destination', default='', desc='Email address to send alerts to', runtime=True), Option(name='smtp_port', type='int', default=465, desc='SMTP port', runtime=True), Option(name='smtp_ssl', type='bool', default=True, desc='Use SSL to connect to SMTP server', runtime=True), Option(name='smtp_user', default='', desc='User to authenticate as', runtime=True), Option(name='smtp_password', default='', desc='Password to authenticate with', runtime=True), Option(name='smtp_sender', default='', desc='SMTP envelope sender', runtime=True), Option(name='smtp_from_name', default='Ceph', desc='Email From: name', runtime=True) ] # These are "native" Ceph options that this module cares about. NATIVE_OPTIONS: List[str] = [] def __init__(self, *args: Any, **kwargs: Any) -> None: super(Alerts, self).__init__(*args, **kwargs) # set up some members to enable the serve() method and shutdown() self.run = True self.event = Event() # ensure config options members are initialized; see config_notify() self.config_notify() self.log.info("Init") if TYPE_CHECKING: self.interval = 60 self.smtp_host = '' self.smtp_destination = '' self.smtp_port = 0 self.smtp_ssl = True self.smtp_user = '' self.smtp_password = '' self.smtp_sender = '' self.smtp_from_name = '' def config_notify(self) -> None: """ This method is called whenever one of our config options is changed. """ # This is some boilerplate that stores MODULE_OPTIONS in a class # member, so that, for instance, the 'emphatic' option is always # available as 'self.emphatic'. for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' mgr option %s = %s', opt['name'], getattr(self, opt['name'])) # Do the same for the native options. for opt in self.NATIVE_OPTIONS: setattr(self, opt, self.get_ceph_option(opt)) self.log.debug(' native option %s = %s', opt, getattr(self, opt)) @CLIReadCommand('alerts send') def send(self) -> HandleCommandResult: """ (re)send alerts immediately """ status = json.loads(self.get('health')['json']) self._send_alert(status, {}) return HandleCommandResult() def _diff(self, last: Dict[str, Any], new: Dict[str, Any]) -> Dict[str, Any]: d: Dict[str, Any] = {} for code, alert in new.get('checks', {}).items(): self.log.debug('new code %s alert %s' % (code, alert)) if code not in last.get('checks', {}): if 'new' not in d: d['new'] = {} d['new'][code] = alert elif (alert['summary'].get('count', 0) > last['checks'][code]['summary'].get('count', 0)): if 'updated' not in d: d['updated'] = {} d['updated'][code] = alert for code, alert in last.get('checks', {}).items(): self.log.debug('old code %s alert %s' % (code, alert)) if code not in new.get('checks', {}): if 'cleared' not in d: d['cleared'] = {} d['cleared'][code] = alert return d def _send_alert(self, status: Dict[str, Any], diff: Dict[str, Any]) -> None: checks = {} if self.smtp_host: r = self._send_alert_smtp(status, diff) if r: for code, alert in r.items(): checks[code] = alert else: self.log.warning( 'Alert is not sent because smtp_host is not configured') self.set_health_checks(checks) def serve(self) -> None: """ This method is called by the mgr when the module starts and can be used for any background activity. """ self.log.info("Starting") last_status: Dict[str, Any] = {} while self.run: # Do some useful background work here. new_status = json.loads(self.get('health')['json']) if new_status != last_status: self.log.debug('last_status %s' % last_status) self.log.debug('new_status %s' % new_status) diff = self._diff(last_status, new_status) self.log.debug('diff %s' % diff) if diff: self._send_alert(new_status, diff) last_status = new_status self.log.debug('Sleeping for %s seconds', self.interval) self.event.wait(self.interval or 60) self.event.clear() def shutdown(self) -> None: """ This method is called by the mgr when the module needs to shut down (i.e., when the serve() function needs to exit). """ self.log.info('Stopping') self.run = False self.event.set() # SMTP def _smtp_format_alert(self, code: str, alert: Dict[str, Any]) -> str: r = '[{sev}] {code}: {summary}\n'.format( code=code, sev=alert['severity'].split('_')[1], summary=alert['summary']['message']) for detail in alert['detail']: r += ' {message}\n'.format(message=detail['message']) return r def _send_alert_smtp(self, status: Dict[str, Any], diff: Dict[str, Any]) -> Optional[Dict[str, Any]]: # message self.log.debug('_send_alert_smtp') message = ('From: {from_name} <{sender}>\n' 'Subject: {status}\n' 'To: {target}\n' '\n' '{status}\n'.format(sender=self.smtp_sender, from_name=self.smtp_from_name, status=status['status'], target=self.smtp_destination)) if 'new' in diff: message += ('\n--- New ---\n') for code, alert in diff['new'].items(): message += self._smtp_format_alert(code, alert) if 'updated' in diff: message += ('\n--- Updated ---\n') for code, alert in diff['updated'].items(): message += self._smtp_format_alert(code, alert) if 'cleared' in diff: message += ('\n--- Cleared ---\n') for code, alert in diff['cleared'].items(): message += self._smtp_format_alert(code, alert) message += ('\n\n=== Full health status ===\n') for code, alert in status['checks'].items(): message += self._smtp_format_alert(code, alert) self.log.debug('message: %s' % message) # send try: if self.smtp_ssl: server: Union[smtplib.SMTP_SSL, smtplib.SMTP] = \ smtplib.SMTP_SSL(self.smtp_host, self.smtp_port) else: server = smtplib.SMTP(self.smtp_host, self.smtp_port) if self.smtp_password: server.login(self.smtp_user, self.smtp_password) server.sendmail(self.smtp_sender, self.smtp_destination, message) server.quit() except Exception as e: return { 'ALERTS_SMTP_ERROR': { 'severity': 'warning', 'summary': 'unable to send alert email', 'count': 1, 'detail': [str(e)] } } self.log.debug('Sent email to %s' % self.smtp_destination) return None
class Module(MgrModule): MODULE_OPTIONS = [ Option(name='address', default='unixgram:///tmp/telegraf.sock'), Option(name='interval', type='secs', default=15) ] ceph_health_mapping = {'HEALTH_OK': 0, 'HEALTH_WARN': 1, 'HEALTH_ERR': 2} @property def config_keys(self) -> Dict[str, OptionValue]: return dict( (o['name'], o.get('default', None)) for o in self.MODULE_OPTIONS) def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) self.event = Event() self.run = True self.fsid: Optional[str] = None self.config: Dict[str, OptionValue] = dict() def get_fsid(self) -> str: if not self.fsid: self.fsid = self.get('mon_map')['fsid'] assert self.fsid is not None return self.fsid def get_pool_stats(self) -> Iterable[Dict[str, Any]]: df = self.get('df') df_types = [ 'bytes_used', 'kb_used', 'dirty', 'rd', 'rd_bytes', 'stored_raw', 'wr', 'wr_bytes', 'objects', 'max_avail', 'quota_objects', 'quota_bytes' ] for df_type in df_types: for pool in df['pools']: yield { 'measurement': 'ceph_pool_stats', 'tags': { 'pool_name': pool['name'], 'pool_id': pool['id'], 'type_instance': df_type, 'fsid': self.get_fsid() }, 'value': pool['stats'][df_type], } def get_daemon_stats(self) -> Iterable[Dict[str, Any]]: for daemon, counters in self.get_all_perf_counters().items(): svc_type, svc_id = daemon.split('.', 1) metadata = self.get_metadata(svc_type, svc_id) if not metadata: continue for path, counter_info in counters.items(): if counter_info['type'] & self.PERFCOUNTER_HISTOGRAM: continue yield { 'measurement': 'ceph_daemon_stats', 'tags': { 'ceph_daemon': daemon, 'type_instance': path, 'host': metadata['hostname'], 'fsid': self.get_fsid() }, 'value': counter_info['value'] } def get_pg_stats(self) -> Dict[str, int]: stats = dict() pg_status = self.get('pg_status') for key in [ 'bytes_total', 'data_bytes', 'bytes_used', 'bytes_avail', 'num_pgs', 'num_objects', 'num_pools' ]: stats[key] = pg_status[key] for state in PG_STATES: stats['num_pgs_{0}'.format(state)] = 0 stats['num_pgs'] = pg_status['num_pgs'] for state in pg_status['pgs_by_state']: states = state['state_name'].split('+') for s in PG_STATES: key = 'num_pgs_{0}'.format(s) if s in states: stats[key] += state['count'] return stats def get_cluster_stats(self) -> Iterable[Dict[str, Any]]: stats = dict() health = json.loads(self.get('health')['json']) stats['health'] = self.ceph_health_mapping.get(health['status']) mon_status = json.loads(self.get('mon_status')['json']) stats['num_mon'] = len(mon_status['monmap']['mons']) stats['mon_election_epoch'] = mon_status['election_epoch'] stats['mon_outside_quorum'] = len(mon_status['outside_quorum']) stats['mon_quorum'] = len(mon_status['quorum']) osd_map = self.get('osd_map') stats['num_osd'] = len(osd_map['osds']) stats['num_pg_temp'] = len(osd_map['pg_temp']) stats['osd_epoch'] = osd_map['epoch'] mgr_map = self.get('mgr_map') stats['mgr_available'] = int(mgr_map['available']) stats['num_mgr_standby'] = len(mgr_map['standbys']) stats['mgr_epoch'] = mgr_map['epoch'] num_up = 0 num_in = 0 for osd in osd_map['osds']: if osd['up'] == 1: num_up += 1 if osd['in'] == 1: num_in += 1 stats['num_osd_up'] = num_up stats['num_osd_in'] = num_in fs_map = self.get('fs_map') stats['num_mds_standby'] = len(fs_map['standbys']) stats['num_fs'] = len(fs_map['filesystems']) stats['mds_epoch'] = fs_map['epoch'] num_mds_up = 0 for fs in fs_map['filesystems']: num_mds_up += len(fs['mdsmap']['up']) stats['num_mds_up'] = num_mds_up stats['num_mds'] = num_mds_up + cast(int, stats['num_mds_standby']) stats.update(self.get_pg_stats()) for key, value in stats.items(): assert value is not None yield { 'measurement': 'ceph_cluster_stats', 'tags': { 'type_instance': key, 'fsid': self.get_fsid() }, 'value': int(value) } def set_config_option(self, option: str, value: str) -> None: if option not in self.config_keys.keys(): raise RuntimeError('{0} is a unknown configuration ' 'option'.format(option)) if option == 'interval': try: interval = int(value) except (ValueError, TypeError): raise RuntimeError('invalid {0} configured. Please specify ' 'a valid integer'.format(option)) if interval < 5: raise RuntimeError( 'interval should be set to at least 5 seconds') self.config[option] = interval else: self.config[option] = value def init_module_config(self) -> None: self.config['address'] = \ self.get_module_option("address", default=self.config_keys['address']) interval = self.get_module_option("interval", default=self.config_keys['interval']) assert interval self.config['interval'] = int(interval) def now(self) -> int: return int(round(time.time() * 1000000000)) def gather_measurements(self) -> Iterable[Dict[str, Any]]: return itertools.chain(self.get_pool_stats(), self.get_daemon_stats(), self.get_cluster_stats()) def send_to_telegraf(self) -> None: url = urlparse(cast(str, self.config['address'])) sock = BaseSocket(url) self.log.debug('Sending data to Telegraf at %s', sock.address) now = self.now() try: with sock as s: for measurement in self.gather_measurements(): self.log.debug(measurement) line = Line(measurement['measurement'], measurement['value'], measurement['tags'], now) self.log.debug(line.to_line_protocol()) s.send(line.to_line_protocol()) except (socket.error, RuntimeError, IOError, OSError): self.log.exception('Failed to send statistics to Telegraf:') except FileNotFoundError: self.log.exception('Failed to open Telegraf at: %s', url.geturl()) def shutdown(self) -> None: self.log.info('Stopping Telegraf module') self.run = False self.event.set() @CLIReadCommand('telegraf config-show') def config_show(self) -> Tuple[int, str, str]: """ Show current configuration """ return 0, json.dumps(self.config), '' @CLICommand('telegraf config-set') def config_set(self, key: str, value: str) -> Tuple[int, str, str]: """ Set a configuration value """ if not value: return -errno.EINVAL, '', 'Value should not be empty or None' self.log.debug('Setting configuration option %s to %s', key, value) self.set_config_option(key, value) self.set_module_option(key, value) return 0, 'Configuration option {0} updated'.format(key), '' @CLICommand('telegraf send') def send(self) -> Tuple[int, str, str]: """ Force sending data to Telegraf """ self.send_to_telegraf() return 0, 'Sending data to Telegraf', '' def self_test(self) -> None: measurements = list(self.gather_measurements()) if len(measurements) == 0: raise RuntimeError('No measurements found') def serve(self) -> None: self.log.info('Starting Telegraf module') self.init_module_config() self.run = True self.log.debug('Waiting 10 seconds before starting') self.event.wait(10) while self.run: start = self.now() self.send_to_telegraf() runtime = (self.now() - start) / 1000000 self.log.debug('Sending data to Telegraf took %d ms', runtime) self.log.debug("Sleeping for %d seconds", self.config['interval']) self.event.wait(cast(int, self.config['interval']))
class Module(MgrModule): # latest (if db does not exist) SCHEMA = """ CREATE TABLE Device ( devid TEXT PRIMARY KEY ) WITHOUT ROWID; CREATE TABLE DeviceHealthMetrics ( time DATETIME DEFAULT (strftime('%s', 'now')), devid TEXT NOT NULL REFERENCES Device (devid), raw_smart TEXT NOT NULL, PRIMARY KEY (time, devid) ); """ SCHEMA_VERSIONED = [ # v1 """ CREATE TABLE Device ( devid TEXT PRIMARY KEY ) WITHOUT ROWID; CREATE TABLE DeviceHealthMetrics ( time DATETIME DEFAULT (strftime('%s', 'now')), devid TEXT NOT NULL REFERENCES Device (devid), raw_smart TEXT NOT NULL, PRIMARY KEY (time, devid) ); """ ] MODULE_OPTIONS = [ Option( name='enable_monitoring', default=True, type='bool', desc='monitor device health metrics', runtime=True, ), Option( name='scrape_frequency', default=86400, type='secs', desc='how frequently to scrape device health metrics', runtime=True, ), Option( name='pool_name', default='device_health_metrics', type='str', desc='name of pool in which to store device health metrics', runtime=True, ), Option( name='retention_period', default=(86400 * 180), type='secs', desc='how long to retain device health metrics', runtime=True, ), Option( name='mark_out_threshold', default=(86400 * 14 * 2), type='secs', desc='automatically mark OSD if it may fail before this long', runtime=True, ), Option( name='warn_threshold', default=(86400 * 14 * 6), type='secs', desc='raise health warning if OSD may fail before this long', runtime=True, ), Option( name='self_heal', default=True, type='bool', desc='preemptively heal cluster around devices that may fail', runtime=True, ), Option( name='sleep_interval', default=600, type='secs', desc='how frequently to wake up and check device health', runtime=True, ), ] def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) # populate options (just until serve() runs) for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], opt['default']) # other self.run = True self.event = Event() # for mypy which does not run the code if TYPE_CHECKING: self.enable_monitoring = True self.scrape_frequency = 0.0 self.pool_name = '' self.device_health_metrics = '' self.retention_period = 0.0 self.mark_out_threshold = 0.0 self.warn_threshold = 0.0 self.self_heal = True self.sleep_interval = 0.0 def is_valid_daemon_name(self, who: str) -> bool: parts = who.split('.') if len(parts) != 2: return False return parts[0] in ('osd', 'mon') @CLICommand('device query-daemon-health-metrics', perm='r') def do_query_daemon_health_metrics(self, who: str) -> Tuple[int, str, str]: ''' Get device health metrics for a given daemon ''' if not self.is_valid_daemon_name(who): return -errno.EINVAL, '', 'not a valid mon or osd daemon name' (daemon_type, daemon_id) = who.split('.') result = CommandResult('') self.send_command(result, daemon_type, daemon_id, json.dumps({ 'prefix': 'smart', 'format': 'json', }), '') return result.wait() @CLIRequiresDB @CLICommand('device scrape-daemon-health-metrics', perm='r') def do_scrape_daemon_health_metrics(self, who: str) -> Tuple[int, str, str]: ''' Scrape and store device health metrics for a given daemon ''' if not self.is_valid_daemon_name(who): return -errno.EINVAL, '', 'not a valid mon or osd daemon name' (daemon_type, daemon_id) = who.split('.') return self.scrape_daemon(daemon_type, daemon_id) @CLIRequiresDB @CLICommand('device scrape-health-metrics', perm='r') def do_scrape_health_metrics(self, devid: Optional[str] = None ) -> Tuple[int, str, str]: ''' Scrape and store device health metrics ''' if devid is None: return self.scrape_all() else: return self.scrape_device(devid) @CLIRequiresDB @CLICommand('device get-health-metrics', perm='r') def do_get_health_metrics( self, devid: str, sample: Optional[str] = None) -> Tuple[int, str, str]: ''' Show stored device metrics for the device ''' return self.show_device_metrics(devid, sample) @CLIRequiresDB @CLICommand('device check-health', perm='rw') def do_check_health(self) -> Tuple[int, str, str]: ''' Check life expectancy of devices ''' return self.check_health() @CLICommand('device monitoring on', perm='rw') def do_monitoring_on(self) -> Tuple[int, str, str]: ''' Enable device health monitoring ''' self.set_module_option('enable_monitoring', True) self.event.set() return 0, '', '' @CLICommand('device monitoring off', perm='rw') def do_monitoring_off(self) -> Tuple[int, str, str]: ''' Disable device health monitoring ''' self.set_module_option('enable_monitoring', False) self.set_health_checks({}) # avoid stuck health alerts return 0, '', '' @CLIRequiresDB @CLICommand('device predict-life-expectancy', perm='r') def do_predict_life_expectancy(self, devid: str) -> Tuple[int, str, str]: ''' Predict life expectancy with local predictor ''' return self.predict_lift_expectancy(devid) def self_test(self) -> None: assert self.db_ready() self.config_notify() osdmap = self.get('osd_map') osd_id = osdmap['osds'][0]['osd'] osdmeta = self.get('osd_metadata') devs = osdmeta.get(str(osd_id), {}).get('device_ids') if devs: devid = devs.split()[0].split('=')[1] self.log.debug(f"getting devid {devid}") (r, before, err) = self.show_device_metrics(devid, None) assert r == 0 self.log.debug(f"before: {before}") (r, out, err) = self.scrape_device(devid) assert r == 0 (r, after, err) = self.show_device_metrics(devid, None) assert r == 0 self.log.debug(f"after: {after}") assert before != after def config_notify(self) -> None: for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name'])) def _legacy_put_device_metrics(self, t: str, devid: str, data: str) -> None: SQL = """ INSERT OR IGNORE INTO DeviceHealthMetrics (time, devid, raw_smart) VALUES (?, ?, ?); """ self._create_device(devid) epoch = self._t2epoch(t) json.loads(data) # valid? self.db.execute(SQL, (epoch, devid, data)) devre = r"[a-zA-Z0-9-]+[_-][a-zA-Z0-9-]+[_-][a-zA-Z0-9-]+" def _load_legacy_object(self, ioctx: rados.Ioctx, oid: str) -> bool: MAX_OMAP = 10000 self.log.debug(f"loading object {oid}") if re.search(self.devre, oid) is None: return False with rados.ReadOpCtx() as op: it, rc = ioctx.get_omap_vals(op, None, None, MAX_OMAP) if rc == 0: ioctx.operate_read_op(op, oid) count = 0 for t, raw_smart in it: self.log.debug(f"putting {oid} {t}") self._legacy_put_device_metrics(t, oid, raw_smart) count += 1 assert count < MAX_OMAP self.log.debug(f"removing object {oid}") ioctx.remove_object(oid) return True def check_legacy_pool(self) -> bool: try: # 'device_health_metrics' is automatically renamed '.mgr' in # create_mgr_pool ioctx = self.rados.open_ioctx(self.MGR_POOL_NAME) except rados.ObjectNotFound: return True if not ioctx: return True done = False with ioctx, self._db_lock, self.db: count = 0 for obj in ioctx.list_objects(): try: if self._load_legacy_object(ioctx, obj.key): count += 1 except json.decoder.JSONDecodeError: pass if count >= 10: break done = count < 10 self.log.debug(f"finished reading legacy pool, complete = {done}") return done def serve(self) -> None: self.log.info("Starting") self.config_notify() last_scrape = None finished_loading_legacy = False while self.run: if self.db_ready() and self.enable_monitoring: self.log.debug('Running') if not finished_loading_legacy: finished_loading_legacy = self.check_legacy_pool() if last_scrape is None: ls = self.get_kv('last_scrape') if ls: try: last_scrape = datetime.strptime(ls, TIME_FORMAT) except ValueError: pass self.log.debug('Last scrape %s', last_scrape) self.check_health() now = datetime.utcnow() if not last_scrape: next_scrape = now else: # align to scrape interval scrape_frequency = self.scrape_frequency or 86400 seconds = (last_scrape - datetime.utcfromtimestamp(0)).total_seconds() seconds -= seconds % scrape_frequency seconds += scrape_frequency next_scrape = datetime.utcfromtimestamp(seconds) if last_scrape: self.log.debug('Last scrape %s, next scrape due %s', last_scrape.strftime(TIME_FORMAT), next_scrape.strftime(TIME_FORMAT)) else: self.log.debug('Last scrape never, next scrape due %s', next_scrape.strftime(TIME_FORMAT)) if now >= next_scrape: self.scrape_all() self.predict_all_devices() last_scrape = now self.set_kv('last_scrape', last_scrape.strftime(TIME_FORMAT)) # sleep sleep_interval = self.sleep_interval or 60 if not finished_loading_legacy: sleep_interval = 2 self.log.debug('Sleeping for %d seconds', sleep_interval) self.event.wait(sleep_interval) self.event.clear() def shutdown(self) -> None: self.log.info('Stopping') self.run = False self.event.set() def scrape_daemon(self, daemon_type: str, daemon_id: str) -> Tuple[int, str, str]: if not self.db_ready(): return -errno.EAGAIN, "", "mgr db not yet available" raw_smart_data = self.do_scrape_daemon(daemon_type, daemon_id) if raw_smart_data: for device, raw_data in raw_smart_data.items(): data = self.extract_smart_features(raw_data) if device and data: self.put_device_metrics(device, data) return 0, "", "" def scrape_all(self) -> Tuple[int, str, str]: if not self.db_ready(): return -errno.EAGAIN, "", "mgr db not yet available" osdmap = self.get("osd_map") assert osdmap is not None did_device = {} ids = [] for osd in osdmap['osds']: ids.append(('osd', str(osd['osd']))) monmap = self.get("mon_map") for mon in monmap['mons']: ids.append(('mon', mon['name'])) for daemon_type, daemon_id in ids: raw_smart_data = self.do_scrape_daemon(daemon_type, daemon_id) if not raw_smart_data: continue for device, raw_data in raw_smart_data.items(): if device in did_device: self.log.debug('skipping duplicate %s' % device) continue did_device[device] = 1 data = self.extract_smart_features(raw_data) if device and data: self.put_device_metrics(device, data) return 0, "", "" def scrape_device(self, devid: str) -> Tuple[int, str, str]: if not self.db_ready(): return -errno.EAGAIN, "", "mgr db not yet available" r = self.get("device " + devid) if not r or 'device' not in r.keys(): return -errno.ENOENT, '', 'device ' + devid + ' not found' daemons = r['device'].get('daemons', []) if not daemons: return (-errno.EAGAIN, '', 'device ' + devid + ' not claimed by any active daemons') (daemon_type, daemon_id) = daemons[0].split('.') raw_smart_data = self.do_scrape_daemon(daemon_type, daemon_id, devid=devid) if raw_smart_data: for device, raw_data in raw_smart_data.items(): data = self.extract_smart_features(raw_data) if device and data: self.put_device_metrics(device, data) return 0, "", "" def do_scrape_daemon(self, daemon_type: str, daemon_id: str, devid: str = '') -> Optional[Dict[str, Any]]: """ :return: a dict, or None if the scrape failed. """ self.log.debug('do_scrape_daemon %s.%s' % (daemon_type, daemon_id)) result = CommandResult('') self.send_command( result, daemon_type, daemon_id, json.dumps({ 'prefix': 'smart', 'format': 'json', 'devid': devid, }), '') r, outb, outs = result.wait() try: return json.loads(outb) except (IndexError, ValueError): self.log.error( "Fail to parse JSON result from daemon {0}.{1} ({2})".format( daemon_type, daemon_id, outb)) return None def _prune_device_metrics(self) -> None: SQL = """ DELETE FROM DeviceHealthMetrics WHERE time < (strftime('%s', 'now') - ?); """ cursor = self.db.execute(SQL, (self.retention_period, )) if cursor.rowcount >= 1: self.log.info(f"pruned {cursor.rowcount} metrics") def _create_device(self, devid: str) -> None: SQL = """ INSERT OR IGNORE INTO Device VALUES (?); """ cursor = self.db.execute(SQL, (devid, )) if cursor.rowcount >= 1: self.log.info(f"created device {devid}") else: self.log.debug(f"device {devid} already exists") def put_device_metrics(self, devid: str, data: Any) -> None: SQL = """ INSERT INTO DeviceHealthMetrics (devid, raw_smart) VALUES (?, ?); """ with self._db_lock, self.db: self._create_device(devid) self.db.execute(SQL, (devid, json.dumps(data))) self._prune_device_metrics() # extract wear level? wear_level = get_ata_wear_level(data) if wear_level is None: wear_level = get_nvme_wear_level(data) dev_data = self.get(f"device {devid}") or {} if wear_level is not None: if dev_data.get(wear_level) != str(wear_level): dev_data["wear_level"] = str(wear_level) self.log.debug(f"updating {devid} wear level to {wear_level}") self.set_device_wear_level(devid, wear_level) else: if "wear_level" in dev_data: del dev_data["wear_level"] self.log.debug(f"removing {devid} wear level") self.set_device_wear_level(devid, -1.0) def _t2epoch(self, t: Optional[str]) -> int: if t is None: return 0 else: return int(datetime.strptime(t, TIME_FORMAT).strftime("%s")) def _get_device_metrics( self, devid: str, sample: Optional[str] = None, min_sample: Optional[str] = None) -> Dict[str, Dict[str, Any]]: res = {} SQL = """ SELECT time, raw_smart FROM DeviceHealthMetrics WHERE devid = ? AND (time = ? OR ? <= time) ORDER BY time DESC; """ isample = self._t2epoch(sample) imin_sample = self._t2epoch(min_sample) self.log.debug(f"_get_device_metrics: {devid} {sample} {min_sample}") with self._db_lock, self.db: cursor = self.db.execute(SQL, (devid, isample, imin_sample)) for row in cursor: t = row['time'] dt = datetime.utcfromtimestamp(t).strftime(TIME_FORMAT) try: res[dt] = json.loads(row['raw_smart']) except (ValueError, IndexError): self.log.debug(f"unable to parse value for {devid}:{t}") pass return res def show_device_metrics(self, devid: str, sample: Optional[str]) -> Tuple[int, str, str]: # verify device exists r = self.get("device " + devid) if not r or 'device' not in r.keys(): return -errno.ENOENT, '', 'device ' + devid + ' not found' # fetch metrics res = self._get_device_metrics(devid, sample=sample) return 0, json.dumps(res, indent=4, sort_keys=True), '' def check_health(self) -> Tuple[int, str, str]: self.log.info('Check health') config = self.get('config') min_in_ratio = float(config.get('mon_osd_min_in_ratio')) mark_out_threshold_td = timedelta(seconds=self.mark_out_threshold) warn_threshold_td = timedelta(seconds=self.warn_threshold) checks: Dict[str, Dict[str, Union[int, str, Sequence[str]]]] = {} health_warnings: Dict[str, List[str]] = { DEVICE_HEALTH: [], DEVICE_HEALTH_IN_USE: [], } devs = self.get("devices") osds_in = {} osds_out = {} now = datetime.utcnow() osdmap = self.get("osd_map") assert osdmap is not None for dev in devs['devices']: if 'life_expectancy_max' not in dev: continue # ignore devices that are not consumed by any daemons if not dev['daemons']: continue if not dev['life_expectancy_max'] or \ dev['life_expectancy_max'] == '0.000000': continue # life_expectancy_(min/max) is in the format of: # '%Y-%m-%dT%H:%M:%S.%f%z', e.g.: # '2019-01-20T21:12:12.000000Z' life_expectancy_max = datetime.strptime(dev['life_expectancy_max'], '%Y-%m-%dT%H:%M:%S.%f%z') self.log.debug('device %s expectancy max %s', dev, life_expectancy_max) if life_expectancy_max - now <= mark_out_threshold_td: if self.self_heal: # dev['daemons'] == ["osd.0","osd.1","osd.2"] if dev['daemons']: osds = [ x for x in dev['daemons'] if x.startswith('osd.') ] osd_ids = map(lambda x: x[4:], osds) for _id in osd_ids: if self.is_osd_in(osdmap, _id): osds_in[_id] = life_expectancy_max else: osds_out[_id] = 1 if life_expectancy_max - now <= warn_threshold_td: # device can appear in more than one location in case # of SCSI multipath device_locations = map(lambda x: x['host'] + ':' + x['dev'], dev['location']) health_warnings[DEVICE_HEALTH].append( '%s (%s); daemons %s; life expectancy between %s and %s' % (dev['devid'], ','.join(device_locations), ','.join( dev.get('daemons', ['none'])), dev['life_expectancy_max'], dev.get('life_expectancy_max', 'unknown'))) # OSD might be marked 'out' (which means it has no # data), however PGs are still attached to it. for _id in osds_out: num_pgs = self.get_osd_num_pgs(_id) if num_pgs > 0: health_warnings[DEVICE_HEALTH_IN_USE].append( 'osd.%s is marked out ' 'but still has %s PG(s)' % (_id, num_pgs)) if osds_in: self.log.debug('osds_in %s' % osds_in) # calculate target in ratio num_osds = len(osdmap['osds']) num_in = len([x for x in osdmap['osds'] if x['in']]) num_bad = len(osds_in) # sort with next-to-fail first bad_osds = sorted(osds_in.items(), key=operator.itemgetter(1)) did = 0 to_mark_out = [] for osd_id, when in bad_osds: ratio = float(num_in - did - 1) / float(num_osds) if ratio < min_in_ratio: final_ratio = float(num_in - num_bad) / float(num_osds) checks[DEVICE_HEALTH_TOOMANY] = { 'severity': 'warning', 'summary': HEALTH_MESSAGES[DEVICE_HEALTH_TOOMANY], 'detail': [ '%d OSDs with failing device(s) would bring "in" ratio to %f < mon_osd_min_in_ratio %f' % (num_bad - did, final_ratio, min_in_ratio) ] } break to_mark_out.append(osd_id) did += 1 if to_mark_out: self.mark_out_etc(to_mark_out) for warning, ls in health_warnings.items(): n = len(ls) if n: checks[warning] = { 'severity': 'warning', 'summary': HEALTH_MESSAGES[warning] % n, 'count': len(ls), 'detail': ls, } self.set_health_checks(checks) return 0, "", "" def is_osd_in(self, osdmap: Dict[str, Any], osd_id: str) -> bool: for osd in osdmap['osds']: if osd_id == str(osd['osd']): return bool(osd['in']) return False def get_osd_num_pgs(self, osd_id: str) -> int: stats = self.get('osd_stats') assert stats is not None for stat in stats['osd_stats']: if osd_id == str(stat['osd']): return stat['num_pgs'] return -1 def mark_out_etc(self, osd_ids: List[str]) -> None: self.log.info('Marking out OSDs: %s' % osd_ids) result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd out', 'format': 'json', 'ids': osd_ids, }), '') r, outb, outs = result.wait() if r != 0: self.log.warning( 'Could not mark OSD %s out. r: [%s], outb: [%s], outs: [%s]', osd_ids, r, outb, outs) for osd_id in osd_ids: result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd primary-affinity', 'format': 'json', 'id': int(osd_id), 'weight': 0.0, }), '') r, outb, outs = result.wait() if r != 0: self.log.warning( 'Could not set osd.%s primary-affinity, ' 'r: [%s], outb: [%s], outs: [%s]', osd_id, r, outb, outs) def extract_smart_features(self, raw: Any) -> Any: # FIXME: extract and normalize raw smartctl --json output and # generate a dict of the fields we care about. return raw def predict_lift_expectancy(self, devid: str) -> Tuple[int, str, str]: plugin_name = '' model = self.get_ceph_option('device_failure_prediction_mode') if cast(str, model).lower() == 'local': plugin_name = 'diskprediction_local' else: return -1, '', 'unable to enable any disk prediction model[local/cloud]' try: can_run, _ = self.remote(plugin_name, 'can_run') if can_run: return self.remote(plugin_name, 'predict_life_expectancy', devid=devid) else: return -1, '', f'{plugin_name} is not available' except Exception: return -1, '', 'unable to invoke diskprediction local or remote plugin' def predict_all_devices(self) -> Tuple[int, str, str]: plugin_name = '' model = self.get_ceph_option('device_failure_prediction_mode') if cast(str, model).lower() == 'local': plugin_name = 'diskprediction_local' else: return -1, '', 'unable to enable any disk prediction model[local/cloud]' try: can_run, _ = self.remote(plugin_name, 'can_run') if can_run: return self.remote(plugin_name, 'predict_all_devices') else: return -1, '', f'{plugin_name} is not available' except Exception: return -1, '', 'unable to invoke diskprediction local or remote plugin' def get_recent_device_metrics( self, devid: str, min_sample: str) -> Dict[str, Dict[str, Any]]: return self._get_device_metrics(devid, min_sample=min_sample) def get_time_format(self) -> str: return TIME_FORMAT
class Module(MgrModule, orchestrator.Orchestrator): """An Orchestrator that uses <Ansible Runner Service> to perform operations """ MODULE_OPTIONS = [ # url:port of the Ansible Runner Service Option(name="server_location", type="str", default=""), # Check server identity (True by default) Option(name="verify_server", type="bool", default=True), # Path to an alternative CA bundle Option(name="ca_bundle", type="str", default="") ] def __init__(self, *args, **kwargs): super(Module, self).__init__(*args, **kwargs) self.run = False self.all_completions = [] self.ar_client = None # TLS certificate and key file names used to connect with the external # Ansible Runner Service self.client_cert_fname = "" self.client_key_fname = "" # used to provide more verbose explanation of errors in status method self.status_message = "" def available(self): """ Check if Ansible Runner service is working """ available = False msg = "" try: if self.ar_client: available = self.ar_client.is_operative() if not available: msg = "No response from Ansible Runner Service" else: msg = "Not possible to initialize connection with Ansible "\ "Runner service." except AnsibleRunnerServiceError as ex: available = False msg = str(ex) # Add more details to the detected problem if self.status_message: msg = "{}:\n{}".format(msg, self.status_message) return (available, msg) def wait(self, completions): """Given a list of Completion instances, progress any which are incomplete. :param completions: list of Completion instances :Returns : True if everything is done. """ # Check progress and update status in each operation # Access completion.status property do the trick for operation in completions: self.log.info("<%s> status:%s", operation, operation.status) completions = filter(lambda x: not x.is_complete, completions) ops_pending = len(completions) self.log.info("Operations pending: %s", ops_pending) return ops_pending == 0 def serve(self): """ Mandatory for standby modules """ self.log.info("Starting Ansible Orchestrator module ...") try: # Verify config options and client certificates self.verify_config() # Ansible runner service client self.ar_client = Client( server_url=self.get_module_option('server_location', ''), verify_server=self.get_module_option('verify_server', True), ca_bundle=self.get_module_option('ca_bundle', ''), client_cert=self.client_cert_fname, client_key=self.client_key_fname, logger=self.log) except AnsibleRunnerServiceError: self.log.exception( "Ansible Runner Service not available. " "Check external server status/TLS identity or " "connection options. If configuration options changed" " try to disable/enable the module.") self.shutdown() return self.run = True def shutdown(self): self.log.info('Stopping Ansible orchestrator module') self.run = False def get_inventory(self, node_filter=None, refresh=False): """ :param : node_filter instance :param : refresh any cached state :Return : A AnsibleReadOperation instance (Completion Object) """ # Create a new read completion object for execute the playbook playbook_operation = PlaybookOperation( client=self.ar_client, playbook=GET_STORAGE_DEVICES_CATALOG_PLAYBOOK, logger=self.log, result_pattern="list storage inventory", params={}) # Assign the process_output function playbook_operation.output_wizard = ProcessInventory( self.ar_client, self.log) playbook_operation.event_filter_list = ["runner_on_ok"] # Execute the playbook to obtain data self._launch_operation(playbook_operation) return playbook_operation def create_osds(self, drive_group, all_hosts): """Create one or more OSDs within a single Drive Group. If no host provided the operation affects all the host in the OSDS role :param drive_group: (orchestrator.DriveGroupSpec), Drive group with the specification of drives to use :param all_hosts : (List[str]), List of hosts where the OSD's must be created """ # Transform drive group specification to Ansible playbook parameters host, osd_spec = dg_2_ansible(drive_group) # Create a new read completion object for execute the playbook playbook_operation = PlaybookOperation(client=self.ar_client, playbook=ADD_OSD_PLAYBOOK, logger=self.log, result_pattern="", params=osd_spec, querystr_dict={"limit": host}) # Filter to get the result playbook_operation.output_wizard = ProcessPlaybookResult( self.ar_client, self.log) playbook_operation.event_filter_list = ["playbook_on_stats"] # Execute the playbook self._launch_operation(playbook_operation) return playbook_operation def remove_osds(self, osd_ids): """Remove osd's. :param osd_ids: List of osd's to be removed (List[int]) """ extravars = { 'osd_to_kill': ",".join([str(osd_id) for osd_id in osd_ids]), 'ireallymeanit': 'yes' } # Create a new read completion object for execute the playbook playbook_operation = PlaybookOperation(client=self.ar_client, playbook=REMOVE_OSD_PLAYBOOK, logger=self.log, result_pattern="", params=extravars) # Filter to get the result playbook_operation.output_wizard = ProcessPlaybookResult( self.ar_client, self.log) playbook_operation.event_filter_list = ["playbook_on_stats"] # Execute the playbook self._launch_operation(playbook_operation) return playbook_operation def get_hosts(self): """Provides a list Inventory nodes """ host_ls_op = ARSOperation(self.ar_client, self.log, URL_GET_HOSTS) host_ls_op.output_wizard = ProcessHostsList(self.ar_client, self.log) return host_ls_op def add_host(self, host): """ Add a host to the Ansible Runner Service inventory in the "orchestrator" group :param host: hostname :returns : orchestrator.WriteCompletion """ url_group = URL_MANAGE_GROUP.format(group_name=ORCHESTRATOR_GROUP) try: # Create the orchestrator default group if not exist. # If exists we ignore the error response dummy_response = self.ar_client.http_post(url_group, "", {}) # Here, the default group exists so... # Prepare the operation for adding the new host add_url = URL_ADD_RM_HOSTS.format( host_name=host, inventory_group=ORCHESTRATOR_GROUP) operations = [HttpOperation(add_url, "post", "", None)] except AnsibleRunnerServiceError as ex: # Problems with the external orchestrator. # Prepare the operation to return the error in a Completion object. self.log.exception("Error checking <orchestrator> group: %s", str(ex)) operations = [HttpOperation(url_group, "post", "", None)] return ARSChangeOperation(self.ar_client, self.log, operations) def remove_host(self, host): """ Remove a host from all the groups in the Ansible Runner Service inventory. :param host: hostname :returns : orchestrator.WriteCompletion """ operations = [] host_groups = [] try: # Get the list of groups where the host is included groups_url = URL_GET_HOST_GROUPS.format(host_name=host) response = self.ar_client.http_get(groups_url) if response.status_code == requests.codes.ok: host_groups = json.loads(response.text)["data"]["groups"] except AnsibleRunnerServiceError: self.log.exception("Error retrieving host groups") if not host_groups: # Error retrieving the groups, prepare the completion object to # execute the problematic operation just to provide the error # to the caller operations = [HttpOperation(groups_url, "get")] else: # Build the operations list operations = list( map( lambda x: HttpOperation( URL_ADD_RM_HOSTS.format(host_name=host, inventory_group=x), "delete"), host_groups)) return ARSChangeOperation(self.ar_client, self.log, operations) def _launch_operation(self, ansible_operation): """Launch the operation and add the operation to the completion objects ongoing :ansible_operation: A read/write ansible operation (completion object) """ # Execute the playbook ansible_operation.execute_playbook() # Add the operation to the list of things ongoing self.all_completions.append(ansible_operation) def verify_config(self): """Verify mandatory settings for the module and provide help to configure properly the orchestrator """ the_crt = None the_key = None # Retrieve TLS content to use and check them # First try to get certiticate and key content for this manager instance # ex: mgr/ansible/mgr0/[crt/key] self.log.info("Tying to use configured specific certificate and key" "files for this server") the_crt = self.get_store("{}/{}".format(self.get_mgr_id(), "crt")) the_key = self.get_store("{}/{}".format(self.get_mgr_id(), "key")) if the_crt is None or the_key is None: # If not possible... try to get generic certificates and key content # ex: mgr/ansible/[crt/key] self.log.warning("Specific tls files for this manager not "\ "configured, trying to use generic files") the_crt = self.get_store("crt") the_key = self.get_store("key") if the_crt is None or the_key is None: self.status_message = "No client certificate configured. Please "\ "set Ansible Runner Service client "\ "certificate and key:\n"\ "ceph ansible set-ssl-certificate-"\ "{key,certificate} -i <file>" self.log.error(self.status_message) return # generate certificate temp files self.client_cert_fname = generate_temp_file("crt", the_crt) self.client_key_fname = generate_temp_file("key", the_key) self.status_message = verify_tls_files(self.client_cert_fname, self.client_key_fname) if self.status_message: self.log.error(self.status_message) return # Check module options if not self.get_module_option("server_location", ""): self.status_message = "No Ansible Runner Service base URL "\ "<server_name>:<port>."\ "Try 'ceph config set mgr mgr/{0}/server_location "\ "<server name/ip>:<port>'".format(self.module_name) self.log.error(self.status_message) return if self.get_module_option("verify_server", True): self.status_message = "TLS server identity verification is enabled"\ " by default.Use 'ceph config set mgr mgr/{0}/verify_server False'"\ "to disable it.Use 'ceph config set mgr mgr/{0}/ca_bundle <path>'"\ "to point an alternative CA bundle path used for TLS server "\ "verification".format(self.module_name) self.log.error(self.status_message) return # Everything ok self.status_message = "" #--------------------------------------------------------------------------- # Ansible Orchestrator self-owned commands #--------------------------------------------------------------------------- @CLIWriteCommand("ansible set-ssl-certificate", "name=mgr_id,type=CephString,req=false") def set_tls_certificate(self, mgr_id=None, inbuf=None): """Load tls certificate in mon k-v store """ if inbuf is None: return -errno.EINVAL, \ 'Please specify the certificate file with "-i" option', '' if mgr_id is not None: self.set_store("{}/crt".format(mgr_id), inbuf) else: self.set_store("crt", inbuf) return 0, "SSL certificate updated", "" @CLIWriteCommand("ansible set-ssl-certificate-key", "name=mgr_id,type=CephString,req=false") def set_tls_certificate_key(self, mgr_id=None, inbuf=None): """Load tls certificate key in mon k-v store """ if inbuf is None: return -errno.EINVAL, \ 'Please specify the certificate key file with "-i" option', \ '' if mgr_id is not None: self.set_store("{}/key".format(mgr_id), inbuf) else: self.set_store("key", inbuf) return 0, "SSL certificate key updated", ""
class Hello(MgrModule): # These are module options we understand. These can be set with # # ceph config set global mgr/hello/<name> <value> # # e.g., # # ceph config set global mgr/hello/place Earth # MODULE_OPTIONS = [ Option(name='place', default='world', desc='a place in the world', runtime=True), # can be updated at runtime (no mgr restart) Option(name='emphatic', type='bool', desc='whether to say it loudly', default=True, runtime=True), Option(name='foo', type='str', enum_allowed=['a', 'b', 'c'], default='a', runtime=True) ] # These are "native" Ceph options that this module cares about. NATIVE_OPTIONS = [ 'mgr_tick_period', ] def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) # set up some members to enable the serve() method and shutdown() self.run = True self.event = Event() # ensure config options members are initialized; see config_notify() self.config_notify() # for mypy which does not run the code if TYPE_CHECKING: self.mgr_tick_period = 0 def config_notify(self) -> None: """ This method is called whenever one of our config options is changed. """ # This is some boilerplate that stores MODULE_OPTIONS in a class # member, so that, for instance, the 'emphatic' option is always # available as 'self.emphatic'. for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' mgr option %s = %s', opt['name'], getattr(self, opt['name'])) # Do the same for the native options. for opt in self.NATIVE_OPTIONS: setattr(self, opt, self.get_ceph_option(opt)) self.log.debug(' native option %s = %s', opt, getattr(self, opt)) # there are CLI commands we implement @CLIReadCommand('hello') def hello(self, person_name: Optional[str] = None) -> HandleCommandResult: """ Say hello """ if person_name is None: who = cast(str, self.get_module_option('place')) else: who = person_name fin = '!' if self.get_module_option('emphatic') else '' return HandleCommandResult(stdout=f'Hello, {who}{fin}') @CLIReadCommand('count') def count(self, num: int) -> HandleCommandResult: """ Do some counting """ ret = 0 out = '' err = '' if num < 1: err = 'That\'s too small a number' ret = -errno.EINVAL elif num > 10: err = 'That\'s too big a number' ret = -errno.EINVAL else: out = 'Hello, I am the count!\n' out += ', '.join([str(x) for x in range(1, num + 1)]) + '!' return HandleCommandResult(retval=ret, stdout=out, stderr=err) def serve(self) -> None: """ This method is called by the mgr when the module starts and can be used for any background activity. """ self.log.info("Starting") while self.run: # Do some useful background work here. # Use mgr_tick_period (default: 2) here just to illustrate # consuming native ceph options. Any real background work # would presumably have some more appropriate frequency. sleep_interval = self.mgr_tick_period self.log.debug('Sleeping for %d seconds', sleep_interval) self.event.wait(sleep_interval) self.event.clear() def shutdown(self) -> None: """ This method is called by the mgr when the module needs to shut down (i.e., when the serve() function needs to exit). """ self.log.info('Stopping') self.run = False self.event.set()
class RookOrchestrator(MgrModule, orchestrator.Orchestrator): """ Writes are a two-phase thing, firstly sending the write to the k8s API (fast) and then waiting for the corresponding change to appear in the Ceph cluster (slow) Right now, we are calling the k8s API synchronously. """ MODULE_OPTIONS: List[Option] = [ # TODO: configure k8s API addr instead of assuming local Option( 'storage_class', type='str', default='local', desc='storage class name for LSO-discovered PVs', ), Option( 'drive_group_interval', type='float', default=300.0, desc= 'interval in seconds between re-application of applied drive_groups', ), ] @staticmethod def can_run() -> Tuple[bool, str]: if not kubernetes_imported: return False, "`kubernetes` python module not found" if not RookEnv().api_version_match(): return False, "Rook version unsupported." return True, '' def available(self) -> Tuple[bool, str, Dict[str, Any]]: if not kubernetes_imported: return False, "`kubernetes` python module not found", {} elif not self._rook_env.has_namespace(): return False, "ceph-mgr not running in Rook cluster", {} try: self.k8s.list_namespaced_pod(self._rook_env.namespace) except ApiException as e: return False, "Cannot reach Kubernetes API: {}".format(e), {} else: return True, "", {} def __init__(self, *args: Any, **kwargs: Any) -> None: super(RookOrchestrator, self).__init__(*args, **kwargs) self._initialized = threading.Event() self._k8s_CoreV1_api: Optional[client.CoreV1Api] = None self._k8s_BatchV1_api: Optional[client.BatchV1Api] = None self._k8s_CustomObjects_api: Optional[client.CustomObjectsApi] = None self._k8s_StorageV1_api: Optional[client.StorageV1Api] = None self._rook_cluster: Optional[RookCluster] = None self._rook_env = RookEnv() self._k8s_AppsV1_api: Optional[client.AppsV1Api] = None self.config_notify() if TYPE_CHECKING: self.storage_class = 'foo' self.drive_group_interval = 10.0 self._load_drive_groups() self._shutdown = threading.Event() def config_notify(self) -> None: """ This method is called whenever one of our config options is changed. TODO: this method should be moved into mgr_module.py """ for opt in self.MODULE_OPTIONS: setattr( self, opt['name'], # type: ignore self.get_module_option(opt['name'])) # type: ignore self.log.debug(' mgr option %s = %s', opt['name'], getattr(self, opt['name'])) # type: ignore assert isinstance(self.storage_class, str) assert isinstance(self.drive_group_interval, float) if self._rook_cluster: self._rook_cluster.storage_class = self.storage_class def shutdown(self) -> None: self._shutdown.set() @property def k8s(self): # type: () -> client.CoreV1Api self._initialized.wait() assert self._k8s_CoreV1_api is not None return self._k8s_CoreV1_api @property def rook_cluster(self): # type: () -> RookCluster self._initialized.wait() assert self._rook_cluster is not None return self._rook_cluster def serve(self) -> None: # For deployed clusters, we should always be running inside # a Rook cluster. For development convenience, also support # running outside (reading ~/.kube config) if self._rook_env.has_namespace(): config.load_incluster_config() else: self.log.warning("DEVELOPMENT ONLY: Reading kube config from ~") config.load_kube_config() # So that I can do port forwarding from my workstation - jcsp from kubernetes.client import configuration configuration.verify_ssl = False self._k8s_CoreV1_api = client.CoreV1Api() self._k8s_BatchV1_api = client.BatchV1Api() self._k8s_CustomObjects_api = client.CustomObjectsApi() self._k8s_StorageV1_api = client.StorageV1Api() self._k8s_AppsV1_api = client.AppsV1Api() try: # XXX mystery hack -- I need to do an API call from # this context, or subsequent API usage from handle_command # fails with SSLError('bad handshake'). Suspect some kind of # thread context setup in SSL lib? self._k8s_CoreV1_api.list_namespaced_pod(self._rook_env.namespace) except ApiException: # Ignore here to make self.available() fail with a proper error message pass assert isinstance(self.storage_class, str) self._rook_cluster = RookCluster(self._k8s_CoreV1_api, self._k8s_BatchV1_api, self._k8s_CustomObjects_api, self._k8s_StorageV1_api, self._k8s_AppsV1_api, self._rook_env, self.storage_class) self._initialized.set() self.config_notify() while not self._shutdown.is_set(): self._apply_drivegroups(list(self._drive_group_map.values())) self._shutdown.wait(self.drive_group_interval) @handle_orch_error def get_inventory( self, host_filter: Optional[orchestrator.InventoryFilter] = None, refresh: bool = False) -> List[orchestrator.InventoryHost]: host_list = None if host_filter and host_filter.hosts: # Explicit host list host_list = host_filter.hosts elif host_filter and host_filter.labels: # TODO: query k8s API to resolve to host list, and pass # it into RookCluster.get_discovered_devices raise NotImplementedError() discovered_devs = self.rook_cluster.get_discovered_devices(host_list) result = [] for host_name, host_devs in discovered_devs.items(): devs = [] for d in host_devs: devs.append(d) result.append( orchestrator.InventoryHost(host_name, inventory.Devices(devs))) return result @handle_orch_error def get_hosts(self): # type: () -> List[orchestrator.HostSpec] return self.rook_cluster.get_hosts() @handle_orch_error def describe_service( self, service_type: Optional[str] = None, service_name: Optional[str] = None, refresh: bool = False) -> List[orchestrator.ServiceDescription]: now = datetime_now() # CephCluster cl = self.rook_cluster.rook_api_get("cephclusters/{0}".format( self.rook_cluster.rook_env.cluster_name)) self.log.debug('CephCluster %s' % cl) image_name = cl['spec'].get('cephVersion', {}).get('image', None) num_nodes = len(self.rook_cluster.get_node_names()) spec = {} if service_type == 'mon' or service_type is None: spec['mon'] = orchestrator.ServiceDescription( spec=ServiceSpec( 'mon', placement=PlacementSpec(count=cl['spec'].get( 'mon', {}).get('count', 1), ), ), size=cl['spec'].get('mon', {}).get('count', 1), container_image_name=image_name, last_refresh=now, ) if service_type == 'mgr' or service_type is None: spec['mgr'] = orchestrator.ServiceDescription( spec=ServiceSpec( 'mgr', placement=PlacementSpec.from_string('count:1'), ), size=1, container_image_name=image_name, last_refresh=now, ) if (service_type == 'crash' or service_type is None and not cl['spec'].get( 'crashCollector', {}).get('disable', False)): spec['crash'] = orchestrator.ServiceDescription( spec=ServiceSpec( 'crash', placement=PlacementSpec.from_string('*'), ), size=num_nodes, container_image_name=image_name, last_refresh=now, ) if service_type == 'mds' or service_type is None: # CephFilesystems all_fs = self.rook_cluster.get_resource("cephfilesystems") for fs in all_fs: svc = 'mds.' + fs['metadata']['name'] if svc in spec: continue # FIXME: we are conflating active (+ standby) with count active = fs['spec'].get('metadataServer', {}).get('activeCount', 1) total_mds = active if fs['spec'].get('metadataServer', {}).get('activeStandby', False): total_mds = active * 2 spec[svc] = orchestrator.ServiceDescription( spec=ServiceSpec( service_type='mds', service_id=fs['metadata']['name'], placement=PlacementSpec(count=active), ), size=total_mds, container_image_name=image_name, last_refresh=now, ) if service_type == 'rgw' or service_type is None: # CephObjectstores all_zones = self.rook_cluster.get_resource("cephobjectstores") for zone in all_zones: svc = 'rgw.' + zone['metadata']['name'] if svc in spec: continue active = zone['spec']['gateway']['instances'] if 'securePort' in zone['spec']['gateway']: ssl = True port = zone['spec']['gateway']['securePort'] else: ssl = False port = zone['spec']['gateway']['port'] or 80 rgw_zone = zone['spec'].get('zone', {}).get('name') or None spec[svc] = orchestrator.ServiceDescription( spec=RGWSpec( service_id=zone['metadata']['name'], rgw_zone=rgw_zone, ssl=ssl, rgw_frontend_port=port, placement=PlacementSpec(count=active), ), size=active, container_image_name=image_name, last_refresh=now, ) if service_type == 'nfs' or service_type is None: # CephNFSes all_nfs = self.rook_cluster.get_resource("cephnfses") nfs_pods = self.rook_cluster.describe_pods('nfs', None, None) for nfs in all_nfs: if nfs['spec']['rados']['pool'] != NFS_POOL_NAME: continue nfs_name = nfs['metadata']['name'] svc = 'nfs.' + nfs_name if svc in spec: continue active = nfs['spec'].get('server', {}).get('active') creation_timestamp = datetime.datetime.strptime( nfs['metadata']['creationTimestamp'], '%Y-%m-%dT%H:%M:%SZ') spec[svc] = orchestrator.ServiceDescription( spec=NFSServiceSpec( service_id=nfs_name, placement=PlacementSpec(count=active), ), size=active, last_refresh=now, running=len([ 1 for pod in nfs_pods if pod['labels']['ceph_nfs'] == nfs_name ]), created=creation_timestamp.astimezone( tz=datetime.timezone.utc)) if service_type == 'osd' or service_type is None: # OSDs # FIXME: map running OSDs back to their respective services... # the catch-all unmanaged all_osds = self.rook_cluster.get_osds() svc = 'osd' spec[svc] = orchestrator.ServiceDescription( spec=DriveGroupSpec( unmanaged=True, service_type='osd', ), size=len(all_osds), last_refresh=now, running=sum(osd.status.phase == 'Running' for osd in all_osds)) # drivegroups for name, dg in self._drive_group_map.items(): spec[f'osd.{name}'] = orchestrator.ServiceDescription( spec=dg, last_refresh=now, size=0, running=0, ) if service_type == 'rbd-mirror' or service_type is None: # rbd-mirrors all_mirrors = self.rook_cluster.get_resource("cephrbdmirrors") for mirror in all_mirrors: logging.warn(mirror) mirror_name = mirror['metadata']['name'] svc = 'rbd-mirror.' + mirror_name if svc in spec: continue spec[svc] = orchestrator.ServiceDescription( spec=ServiceSpec( service_id=mirror_name, service_type="rbd-mirror", placement=PlacementSpec(count=1), ), size=1, last_refresh=now, ) for dd in self._list_daemons(): if dd.service_name() not in spec: continue service = spec[dd.service_name()] service.running += 1 if not service.container_image_id: service.container_image_id = dd.container_image_id if not service.container_image_name: service.container_image_name = dd.container_image_name if service.last_refresh is None or not dd.last_refresh or dd.last_refresh < service.last_refresh: service.last_refresh = dd.last_refresh if service.created is None or dd.created is None or dd.created < service.created: service.created = dd.created return [v for k, v in spec.items()] @handle_orch_error def list_daemons( self, service_name: Optional[str] = None, daemon_type: Optional[str] = None, daemon_id: Optional[str] = None, host: Optional[str] = None, refresh: bool = False) -> List[orchestrator.DaemonDescription]: return self._list_daemons(service_name=service_name, daemon_type=daemon_type, daemon_id=daemon_id, host=host, refresh=refresh) def _list_daemons( self, service_name: Optional[str] = None, daemon_type: Optional[str] = None, daemon_id: Optional[str] = None, host: Optional[str] = None, refresh: bool = False) -> List[orchestrator.DaemonDescription]: pods = self.rook_cluster.describe_pods(daemon_type, daemon_id, host) self.log.debug('pods %s' % pods) result = [] for p in pods: sd = orchestrator.DaemonDescription() sd.hostname = p['hostname'] sd.daemon_type = p['labels']['app'].replace('rook-ceph-', '') status = { 'Pending': orchestrator.DaemonDescriptionStatus.error, 'Running': orchestrator.DaemonDescriptionStatus.running, 'Succeeded': orchestrator.DaemonDescriptionStatus.stopped, 'Failed': orchestrator.DaemonDescriptionStatus.error, 'Unknown': orchestrator.DaemonDescriptionStatus.error, }[p['phase']] sd.status = status sd.status_desc = p['phase'] if 'ceph_daemon_id' in p['labels']: sd.daemon_id = p['labels']['ceph_daemon_id'] elif 'ceph-osd-id' in p['labels']: sd.daemon_id = p['labels']['ceph-osd-id'] else: # Unknown type -- skip it continue if service_name is not None and service_name != sd.service_name(): continue sd.container_image_name = p['container_image_name'] sd.container_image_id = p['container_image_id'] sd.created = p['created'] sd.last_configured = p['created'] sd.last_deployed = p['created'] sd.started = p['started'] sd.last_refresh = p['refreshed'] result.append(sd) return result def _get_pool_params(self) -> Tuple[int, str]: num_replicas = self.get_ceph_option('osd_pool_default_size') assert type(num_replicas) is int leaf_type_id = self.get_ceph_option('osd_crush_chooseleaf_type') assert type(leaf_type_id) is int crush = self.get('osd_map_crush') leaf_type = 'host' for t in crush['types']: if t['type_id'] == leaf_type_id: leaf_type = t['name'] break return num_replicas, leaf_type @handle_orch_error def remove_service(self, service_name: str, force: bool = False) -> str: if service_name == 'rbd-mirror': return self.rook_cluster.rm_service('cephrbdmirrors', 'default-rbd-mirror') service_type, service_id = service_name.split('.', 1) if service_type == 'mds': return self.rook_cluster.rm_service('cephfilesystems', service_id) elif service_type == 'rgw': return self.rook_cluster.rm_service('cephobjectstores', service_id) elif service_type == 'nfs': ret, out, err = self.mon_command({'prefix': 'auth ls'}) matches = re.findall(rf'client\.nfs-ganesha\.{service_id}\..*', out) for match in matches: self.check_mon_command({'prefix': 'auth rm', 'entity': match}) return self.rook_cluster.rm_service('cephnfses', service_id) elif service_type == 'rbd-mirror': return self.rook_cluster.rm_service('cephrbdmirrors', service_id) elif service_type == 'osd': if service_id in self._drive_group_map: del self._drive_group_map[service_id] self._save_drive_groups() return f'Removed {service_name}' elif service_type == 'ingress': self.log.info("{0} service '{1}' does not exist".format( 'ingress', service_id)) return 'The Rook orchestrator does not currently support ingress' else: raise orchestrator.OrchestratorError( f'Service type {service_type} not supported') def zap_device(self, host: str, path: str) -> OrchResult[str]: try: self.rook_cluster.create_zap_job(host, path) except Exception as e: logging.error(e) return OrchResult( None, Exception("Unable to zap device: " + str(e.with_traceback(None)))) return OrchResult(f'{path} on {host} zapped') @handle_orch_error def apply_mon(self, spec): # type: (ServiceSpec) -> str if spec.placement.hosts or spec.placement.label: raise RuntimeError("Host list or label is not supported by rook.") return self.rook_cluster.update_mon_count(spec.placement.count) def apply_rbd_mirror(self, spec: ServiceSpec) -> OrchResult[str]: try: self.rook_cluster.rbd_mirror(spec) return OrchResult("Success") except Exception as e: return OrchResult(None, e) @handle_orch_error def apply_mds(self, spec): # type: (ServiceSpec) -> str num_replicas, leaf_type = self._get_pool_params() return self.rook_cluster.apply_filesystem(spec, num_replicas, leaf_type) @handle_orch_error def apply_rgw(self, spec): # type: (RGWSpec) -> str num_replicas, leaf_type = self._get_pool_params() return self.rook_cluster.apply_objectstore(spec, num_replicas, leaf_type) @handle_orch_error def apply_nfs(self, spec): # type: (NFSServiceSpec) -> str try: return self.rook_cluster.apply_nfsgw(spec, self) except Exception as e: logging.error(e) return "Unable to create NFS daemon, check logs for more traceback\n" + str( e.with_traceback(None)) @handle_orch_error def remove_daemons(self, names: List[str]) -> List[str]: return self.rook_cluster.remove_pods(names) def apply_drivegroups( self, specs: List[DriveGroupSpec]) -> OrchResult[List[str]]: for drive_group in specs: self._drive_group_map[str(drive_group.service_id)] = drive_group self._save_drive_groups() return OrchResult(self._apply_drivegroups(specs)) def _apply_drivegroups(self, ls: List[DriveGroupSpec]) -> List[str]: all_hosts = raise_if_exception(self.get_hosts()) result_list: List[str] = [] for drive_group in ls: matching_hosts = drive_group.placement.filter_matching_hosts( lambda label=None, as_hostspec=None: all_hosts) if not self.rook_cluster.node_exists(matching_hosts[0]): raise RuntimeError("Node '{0}' is not in the Kubernetes " "cluster".format(matching_hosts)) # Validate whether cluster CRD can accept individual OSD # creations (i.e. not useAllDevices) if not self.rook_cluster.can_create_osd(): raise RuntimeError("Rook cluster configuration does not " "support OSD creation.") result_list.append( self.rook_cluster.add_osds(drive_group, matching_hosts)) return result_list def _load_drive_groups(self) -> None: stored_drive_group = self.get_store("drive_group_map") self._drive_group_map: Dict[str, DriveGroupSpec] = {} if stored_drive_group: for name, dg in json.loads(stored_drive_group).items(): try: self._drive_group_map[name] = DriveGroupSpec.from_json(dg) except ValueError as e: self.log.error( f'Failed to load drive group {name} ({dg}): {e}') def _save_drive_groups(self) -> None: json_drive_group_map = { name: dg.to_json() for name, dg in self._drive_group_map.items() } self.set_store("drive_group_map", json.dumps(json_drive_group_map)) def remove_osds(self, osd_ids: List[str], replace: bool = False, force: bool = False, zap: bool = False) -> OrchResult[str]: assert self._rook_cluster is not None if zap: raise RuntimeError( "Rook does not support zapping devices during OSD removal.") res = self._rook_cluster.remove_osds(osd_ids, replace, force, self.mon_command) return OrchResult(res) def add_host_label(self, host: str, label: str) -> OrchResult[str]: return self.rook_cluster.add_host_label(host, label) def remove_host_label(self, host: str, label: str) -> OrchResult[str]: return self.rook_cluster.remove_host_label(host, label) """ @handle_orch_error def create_osds(self, drive_group): # type: (DriveGroupSpec) -> str # Creates OSDs from a drive group specification. # $: ceph orch osd create -i <dg.file> # The drivegroup file must only contain one spec at a time. # targets = [] # type: List[str] if drive_group.data_devices and drive_group.data_devices.paths: targets += [d.path for d in drive_group.data_devices.paths] if drive_group.data_directories: targets += drive_group.data_directories all_hosts = raise_if_exception(self.get_hosts()) matching_hosts = drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: all_hosts) assert len(matching_hosts) == 1 if not self.rook_cluster.node_exists(matching_hosts[0]): raise RuntimeError("Node '{0}' is not in the Kubernetes " "cluster".format(matching_hosts)) # Validate whether cluster CRD can accept individual OSD # creations (i.e. not useAllDevices) if not self.rook_cluster.can_create_osd(): raise RuntimeError("Rook cluster configuration does not " "support OSD creation.") return self.rook_cluster.add_osds(drive_group, matching_hosts) # TODO: this was the code to update the progress reference: @handle_orch_error def has_osds(matching_hosts: List[str]) -> bool: # Find OSD pods on this host pod_osd_ids = set() pods = self.k8s.list_namespaced_pod(self._rook_env.namespace, label_selector="rook_cluster={},app=rook-ceph-osd".format(self._rook_env.cluster_name), field_selector="spec.nodeName={0}".format( matching_hosts[0] )).items for p in pods: pod_osd_ids.add(int(p.metadata.labels['ceph-osd-id'])) self.log.debug('pod_osd_ids={0}'.format(pod_osd_ids)) found = [] osdmap = self.get("osd_map") for osd in osdmap['osds']: osd_id = osd['osd'] if osd_id not in pod_osd_ids: continue metadata = self.get_metadata('osd', "%s" % osd_id) if metadata and metadata['devices'] in targets: found.append(osd_id) else: self.log.info("ignoring osd {0} {1}".format( osd_id, metadata['devices'] if metadata else 'DNE' )) return found is not None """ @handle_orch_error def blink_device_light( self, ident_fault: str, on: bool, locs: List[orchestrator.DeviceLightLoc]) -> List[str]: return self.rook_cluster.blink_light(ident_fault, on, locs)
class Module(MgrModule): MODULE_OPTIONS = [ Option(name='subtree', type='str', default='rack', desc='CRUSH level for which to create a local pool', runtime=True), Option(name='failure_domain', type='str', default='host', desc='failure domain for any created local pool', runtime=True), Option(name='min_size', type='int', desc='default min_size for any created local pool', runtime=True), Option(name='num_rep', type='int', default=3, desc='default replica count for any created local pool', runtime=True), Option(name='pg_num', type='int', default=128, desc='default pg_num for any created local pool', runtime=True), Option(name='prefix', type='str', default='', desc='name prefix for any created local pool', runtime=True), ] def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) self.serve_event = threading.Event() def notify(self, notify_type: str, notify_id: str) -> None: if notify_type == 'osd_map': self.handle_osd_map() def handle_osd_map(self) -> None: """ Check pools on each OSDMap change """ subtree_type = cast(str, self.get_module_option('subtree')) failure_domain = self.get_module_option('failure_domain') pg_num = self.get_module_option('pg_num') num_rep = self.get_module_option('num_rep') min_size = self.get_module_option('min_size') prefix = cast( str, self.get_module_option('prefix')) or 'by-' + subtree_type + '-' osdmap = self.get("osd_map") lpools = [] for pool in osdmap['pools']: if pool['pool_name'].find(prefix) == 0: lpools.append(pool['pool_name']) self.log.debug('localized pools = %s', lpools) subtrees = [] tree = self.get('osd_map_tree') for node in tree['nodes']: if node['type'] == subtree_type: subtrees.append(node['name']) pool_name = prefix + node['name'] if pool_name not in lpools: self.log.info('Creating localized pool %s', pool_name) # result = CommandResult("") self.send_command( result, "mon", "", json.dumps({ "prefix": "osd crush rule create-replicated", "format": "json", "name": pool_name, "root": node['name'], "type": failure_domain, }), "") r, outb, outs = result.wait() result = CommandResult("") self.send_command( result, "mon", "", json.dumps({ "prefix": "osd pool create", "format": "json", "pool": pool_name, 'rule': pool_name, "pool_type": 'replicated', 'pg_num': pg_num, }), "") r, outb, outs = result.wait() result = CommandResult("") self.send_command( result, "mon", "", json.dumps({ "prefix": "osd pool set", "format": "json", "pool": pool_name, 'var': 'size', "val": str(num_rep), }), "") r, outb, outs = result.wait() if min_size: result = CommandResult("") self.send_command( result, "mon", "", json.dumps({ "prefix": "osd pool set", "format": "json", "pool": pool_name, 'var': 'min_size', "val": str(min_size), }), "") r, outb, outs = result.wait() # TODO remove pools for hosts that don't exist? def serve(self) -> None: self.handle_osd_map() self.serve_event.wait() self.serve_event.clear() def shutdown(self) -> None: self.serve_event.set()
class Module(MgrModule): metadata_keys = [ "arch", "ceph_version", "os", "cpu", "kernel_description", "kernel_version", "distro_description", "distro" ] MODULE_OPTIONS = [ Option(name='url', type='str', default='https://telemetry.ceph.com/report'), Option(name='device_url', type='str', default='https://telemetry.ceph.com/device'), Option(name='enabled', type='bool', default=False), Option(name='last_opt_revision', type='int', default=1), Option(name='leaderboard', type='bool', default=False), Option(name='description', type='str', default=None), Option(name='contact', type='str', default=None), Option(name='organization', type='str', default=None), Option(name='proxy', type='str', default=None), Option(name='interval', type='int', default=24, min=8), Option(name='channel_basic', type='bool', default=True, desc='Share basic cluster information (size, version)'), Option( name='channel_ident', type='bool', default=False, desc= 'Share a user-provided description and/or contact email for the cluster' ), Option( name='channel_crash', type='bool', default=True, desc= 'Share metadata about Ceph daemon crashes (version, stack straces, etc)' ), Option( name='channel_device', type='bool', default=True, desc= ('Share device health metrics ' '(e.g., SMART data, minus potentially identifying info like serial numbers)' )), ] @property def config_keys(self) -> Dict[str, OptionValue]: return dict( (o['name'], o.get('default', None)) for o in self.MODULE_OPTIONS) def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) self.event = Event() self.run = False self.last_upload: Optional[int] = None self.last_report: Dict[str, Any] = dict() self.report_id: Optional[str] = None self.salt: Optional[str] = None # for mypy which does not run the code if TYPE_CHECKING: self.url = '' self.device_url = '' self.enabled = False self.last_opt_revision = 0 self.leaderboard = '' self.interval = 0 self.proxy = '' self.channel_basic = True self.channel_ident = False self.channel_crash = True self.channel_device = True def config_notify(self) -> None: for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name'])) # wake up serve() thread self.event.set() def load(self) -> None: last_upload = self.get_store('last_upload', None) if last_upload is None: self.last_upload = None else: self.last_upload = int(last_upload) report_id = self.get_store('report_id', None) if report_id is None: self.report_id = str(uuid.uuid4()) self.set_store('report_id', self.report_id) else: self.report_id = report_id salt = self.get_store('salt', None) if salt is None: self.salt = str(uuid.uuid4()) self.set_store('salt', self.salt) else: self.salt = salt def gather_osd_metadata( self, osd_map: Dict[str, List[Dict[str, int]]]) -> Dict[str, Dict[str, int]]: keys = ["osd_objectstore", "rotational"] keys += self.metadata_keys metadata: Dict[str, Dict[str, int]] = dict() for key in keys: metadata[key] = defaultdict(int) for osd in osd_map['osds']: res = self.get_metadata('osd', str(osd['osd'])) if res is None: self.log.debug('Could not get metadata for osd.%s' % str(osd['osd'])) continue for k, v in res.items(): if k not in keys: continue metadata[k][v] += 1 return metadata def gather_mon_metadata( self, mon_map: Dict[str, List[Dict[str, str]]]) -> Dict[str, Dict[str, int]]: keys = list() keys += self.metadata_keys metadata: Dict[str, Dict[str, int]] = dict() for key in keys: metadata[key] = defaultdict(int) for mon in mon_map['mons']: res = self.get_metadata('mon', mon['name']) if res is None: self.log.debug('Could not get metadata for mon.%s' % (mon['name'])) continue for k, v in res.items(): if k not in keys: continue metadata[k][v] += 1 return metadata def gather_crush_info( self ) -> Dict[str, Union[int, bool, List[int], Dict[str, int], Dict[int, int]]]: osdmap = self.get_osdmap() crush_raw = osdmap.get_crush() crush = crush_raw.dump() BucketKeyT = TypeVar('BucketKeyT', int, str) def inc(d: Dict[BucketKeyT, int], k: BucketKeyT) -> None: if k in d: d[k] += 1 else: d[k] = 1 device_classes: Dict[str, int] = {} for dev in crush['devices']: inc(device_classes, dev.get('class', '')) bucket_algs: Dict[str, int] = {} bucket_types: Dict[str, int] = {} bucket_sizes: Dict[int, int] = {} for bucket in crush['buckets']: if '~' in bucket['name']: # ignore shadow buckets continue inc(bucket_algs, bucket['alg']) inc(bucket_types, bucket['type_id']) inc(bucket_sizes, len(bucket['items'])) return { 'num_devices': len(crush['devices']), 'num_types': len(crush['types']), 'num_buckets': len(crush['buckets']), 'num_rules': len(crush['rules']), 'device_classes': list(device_classes.values()), 'tunables': crush['tunables'], 'compat_weight_set': '-1' in crush['choose_args'], 'num_weight_sets': len(crush['choose_args']), 'bucket_algs': bucket_algs, 'bucket_sizes': bucket_sizes, 'bucket_types': bucket_types, } def gather_configs(self) -> Dict[str, List[str]]: # cluster config options cluster = set() r, outb, outs = self.mon_command({ 'prefix': 'config dump', 'format': 'json' }) if r != 0: return {} try: dump = json.loads(outb) except json.decoder.JSONDecodeError: return {} for opt in dump: name = opt.get('name') if name: cluster.add(name) # daemon-reported options (which may include ceph.conf) active = set() ls = self.get("modified_config_options") for opt in ls.get('options', {}): active.add(opt) return { 'cluster_changed': sorted(list(cluster)), 'active_changed': sorted(list(active)), } def gather_crashinfo(self) -> List[Dict[str, str]]: crashlist: List[Dict[str, str]] = list() errno, crashids, err = self.remote('crash', 'ls') if errno: return crashlist for crashid in crashids.split(): errno, crashinfo, err = self.remote('crash', 'do_info', crashid) if errno: continue c = json.loads(crashinfo) # redact hostname del c['utsname_hostname'] # entity_name might have more than one '.', beware (etype, eid) = c.get('entity_name', '').split('.', 1) m = hashlib.sha1() assert self.salt m.update(self.salt.encode('utf-8')) m.update(eid.encode('utf-8')) m.update(self.salt.encode('utf-8')) c['entity_name'] = etype + '.' + m.hexdigest() # redact final line of python tracebacks, as the exception # payload may contain identifying information if 'mgr_module' in c: c['backtrace'][-1] = '<redacted>' crashlist.append(c) return crashlist def get_active_channels(self) -> List[str]: r = [] if self.channel_basic: r.append('basic') if self.channel_crash: r.append('crash') if self.channel_device: r.append('device') if self.channel_ident: r.append('ident') return r def gather_device_report(self) -> Dict[str, Dict[str, Dict[str, str]]]: try: time_format = self.remote('devicehealth', 'get_time_format') except Exception: return {} cutoff = datetime.utcnow() - timedelta(hours=self.interval * 2) min_sample = cutoff.strftime(time_format) devices = self.get('devices')['devices'] # anon-host-id -> anon-devid -> { timestamp -> record } res: Dict[str, Dict[str, Dict[str, str]]] = {} for d in devices: devid = d['devid'] try: # this is a map of stamp -> {device info} m = self.remote('devicehealth', 'get_recent_device_metrics', devid, min_sample) except Exception: continue # anonymize host id try: host = d['location'][0]['host'] except KeyError: continue anon_host = self.get_store('host-id/%s' % host) if not anon_host: anon_host = str(uuid.uuid1()) self.set_store('host-id/%s' % host, anon_host) serial = None for dev, rep in m.items(): rep['host_id'] = anon_host if serial is None and 'serial_number' in rep: serial = rep['serial_number'] # anonymize device id anon_devid = self.get_store('devid-id/%s' % devid) if not anon_devid: # ideally devid is 'vendor_model_serial', # but can also be 'model_serial', 'serial' if '_' in devid: anon_devid = f"{devid.rsplit('_', 1)[0]}_{uuid.uuid1()}" else: anon_devid = str(uuid.uuid1()) self.set_store('devid-id/%s' % devid, anon_devid) self.log.info('devid %s / %s, host %s / %s' % (devid, anon_devid, host, anon_host)) # anonymize the smartctl report itself if serial: m_str = json.dumps(m) m = json.loads(m_str.replace(serial, 'deleted')) if anon_host not in res: res[anon_host] = {} res[anon_host][anon_devid] = m return res def get_latest(self, daemon_type: str, daemon_name: str, stat: str) -> int: data = self.get_counter(daemon_type, daemon_name, stat)[stat] if data: return data[-1][1] else: return 0 def compile_report(self, channels: Optional[List[str]] = None) -> Dict[str, Any]: if not channels: channels = self.get_active_channels() report = { 'leaderboard': self.leaderboard, 'report_version': 1, 'report_timestamp': datetime.utcnow().isoformat(), 'report_id': self.report_id, 'channels': channels, 'channels_available': ALL_CHANNELS, 'license': LICENSE, } if 'ident' in channels: for option in ['description', 'contact', 'organization']: report[option] = getattr(self, option) if 'basic' in channels: mon_map = self.get('mon_map') osd_map = self.get('osd_map') service_map = self.get('service_map') fs_map = self.get('fs_map') df = self.get('df') report['created'] = mon_map['created'] # mons v1_mons = 0 v2_mons = 0 ipv4_mons = 0 ipv6_mons = 0 for mon in mon_map['mons']: for a in mon['public_addrs']['addrvec']: if a['type'] == 'v2': v2_mons += 1 elif a['type'] == 'v1': v1_mons += 1 if a['addr'].startswith('['): ipv6_mons += 1 else: ipv4_mons += 1 report['mon'] = { 'count': len(mon_map['mons']), 'features': mon_map['features'], 'min_mon_release': mon_map['min_mon_release'], 'v1_addr_mons': v1_mons, 'v2_addr_mons': v2_mons, 'ipv4_addr_mons': ipv4_mons, 'ipv6_addr_mons': ipv6_mons, } report['config'] = self.gather_configs() # pools rbd_num_pools = 0 rbd_num_images_by_pool = [] rbd_mirroring_by_pool = [] num_pg = 0 report['pools'] = list() for pool in osd_map['pools']: num_pg += pool['pg_num'] ec_profile = {} if pool['erasure_code_profile']: orig = osd_map['erasure_code_profiles'].get( pool['erasure_code_profile'], {}) ec_profile = { k: orig[k] for k in orig.keys() if k in [ 'k', 'm', 'plugin', 'technique', 'crush-failure-domain', 'l' ] } cast(List[Dict[str, Any]], report['pools']).append({ 'pool': pool['pool'], 'pg_num': pool['pg_num'], 'pgp_num': pool['pg_placement_num'], 'size': pool['size'], 'min_size': pool['min_size'], 'pg_autoscale_mode': pool['pg_autoscale_mode'], 'target_max_bytes': pool['target_max_bytes'], 'target_max_objects': pool['target_max_objects'], 'type': ['', 'replicated', '', 'erasure'][pool['type']], 'erasure_code_profile': ec_profile, 'cache_mode': pool['cache_mode'], }) if 'rbd' in pool['application_metadata']: rbd_num_pools += 1 ioctx = self.rados.open_ioctx(pool['pool_name']) rbd_num_images_by_pool.append( sum(1 for _ in rbd.RBD().list2(ioctx))) rbd_mirroring_by_pool.append(rbd.RBD().mirror_mode_get( ioctx) != rbd.RBD_MIRROR_MODE_DISABLED) report['rbd'] = { 'num_pools': rbd_num_pools, 'num_images_by_pool': rbd_num_images_by_pool, 'mirroring_by_pool': rbd_mirroring_by_pool } # osds cluster_network = False for osd in osd_map['osds']: if osd['up'] and not cluster_network: front_ip = osd['public_addrs']['addrvec'][0]['addr'].split( ':')[0] back_ip = osd['cluster_addrs']['addrvec'][0]['addr'].split( ':')[0] if front_ip != back_ip: cluster_network = True report['osd'] = { 'count': len(osd_map['osds']), 'require_osd_release': osd_map['require_osd_release'], 'require_min_compat_client': osd_map['require_min_compat_client'], 'cluster_network': cluster_network, } # crush report['crush'] = self.gather_crush_info() # cephfs report['fs'] = { 'count': len(fs_map['filesystems']), 'feature_flags': fs_map['feature_flags'], 'num_standby_mds': len(fs_map['standbys']), 'filesystems': [], } num_mds = len(fs_map['standbys']) for fsm in fs_map['filesystems']: fs = fsm['mdsmap'] num_sessions = 0 cached_ino = 0 cached_dn = 0 cached_cap = 0 subtrees = 0 rfiles = 0 rbytes = 0 rsnaps = 0 for gid, mds in fs['info'].items(): num_sessions += self.get_latest( 'mds', mds['name'], 'mds_sessions.session_count') cached_ino += self.get_latest('mds', mds['name'], 'mds_mem.ino') cached_dn += self.get_latest('mds', mds['name'], 'mds_mem.dn') cached_cap += self.get_latest('mds', mds['name'], 'mds_mem.cap') subtrees += self.get_latest('mds', mds['name'], 'mds.subtrees') if mds['rank'] == 0: rfiles = self.get_latest('mds', mds['name'], 'mds.root_rfiles') rbytes = self.get_latest('mds', mds['name'], 'mds.root_rbytes') rsnaps = self.get_latest('mds', mds['name'], 'mds.root_rsnaps') report['fs']['filesystems'].append({ # type: ignore 'max_mds': fs['max_mds'], 'ever_allowed_features': fs['ever_allowed_features'], 'explicitly_allowed_features': fs['explicitly_allowed_features'], 'num_in': len(fs['in']), 'num_up': len(fs['up']), 'num_standby_replay': len( [mds for gid, mds in fs['info'].items() if mds['state'] == 'up:standby-replay']), 'num_mds': len(fs['info']), 'num_sessions': num_sessions, 'cached_inos': cached_ino, 'cached_dns': cached_dn, 'cached_caps': cached_cap, 'cached_subtrees': subtrees, 'balancer_enabled': len(fs['balancer']) > 0, 'num_data_pools': len(fs['data_pools']), 'standby_count_wanted': fs['standby_count_wanted'], 'approx_ctime': fs['created'][0:7], 'files': rfiles, 'bytes': rbytes, 'snaps': rsnaps, }) num_mds += len(fs['info']) report['fs']['total_num_mds'] = num_mds # type: ignore # daemons report['metadata'] = dict(osd=self.gather_osd_metadata(osd_map), mon=self.gather_mon_metadata(mon_map)) # host counts servers = self.list_servers() self.log.debug('servers %s' % servers) hosts = { 'num': len([h for h in servers if h['hostname']]), } for t in ['mon', 'mds', 'osd', 'mgr']: nr_services = sum(1 for host in servers if any( service for service in cast(List[ServiceInfoT], host['services']) if service['type'] == t)) hosts['num_with_' + t] = nr_services report['hosts'] = hosts report['usage'] = { 'pools': len(df['pools']), 'pg_num': num_pg, 'total_used_bytes': df['stats']['total_used_bytes'], 'total_bytes': df['stats']['total_bytes'], 'total_avail_bytes': df['stats']['total_avail_bytes'] } services: DefaultDict[str, int] = defaultdict(int) for key, value in service_map['services'].items(): services[key] += 1 if key == 'rgw': rgw = {} zones = set() zonegroups = set() frontends = set() count = 0 d = value.get('daemons', dict()) for k, v in d.items(): if k == 'summary' and v: rgw[k] = v elif isinstance(v, dict) and 'metadata' in v: count += 1 zones.add(v['metadata']['zone_id']) zonegroups.add(v['metadata']['zonegroup_id']) frontends.add(v['metadata']['frontend_type#0']) # we could actually iterate over all the keys of # the dict and check for how many frontends there # are, but it is unlikely that one would be running # more than 2 supported ones f2 = v['metadata'].get('frontend_type#1', None) if f2: frontends.add(f2) rgw['count'] = count rgw['zones'] = len(zones) rgw['zonegroups'] = len(zonegroups) rgw['frontends'] = list( frontends) # sets aren't json-serializable report['rgw'] = rgw report['services'] = services try: report['balancer'] = self.remote('balancer', 'gather_telemetry') except ImportError: report['balancer'] = {'active': False} if 'crash' in channels: report['crashes'] = self.gather_crashinfo() # NOTE: We do not include the 'device' channel in this report; it is # sent to a different endpoint. return report def _try_post(self, what: str, url: str, report: Dict[str, Dict[str, str]]) -> Optional[str]: self.log.info('Sending %s to: %s' % (what, url)) proxies = dict() if self.proxy: self.log.info('Send using HTTP(S) proxy: %s', self.proxy) proxies['http'] = self.proxy proxies['https'] = self.proxy try: resp = requests.put(url=url, json=report, proxies=proxies) resp.raise_for_status() except Exception as e: fail_reason = 'Failed to send %s to %s: %s' % (what, url, str(e)) self.log.error(fail_reason) return fail_reason return None class EndPoint(enum.Enum): ceph = 'ceph' device = 'device' def send( self, report: Dict[str, Dict[str, str]], endpoint: Optional[List[EndPoint]] = None) -> Tuple[int, str, str]: if not endpoint: endpoint = [self.EndPoint.ceph, self.EndPoint.device] failed = [] success = [] self.log.debug('Send endpoints %s' % endpoint) for e in endpoint: if e == self.EndPoint.ceph: fail_reason = self._try_post('ceph report', self.url, report) if fail_reason: failed.append(fail_reason) else: now = int(time.time()) self.last_upload = now self.set_store('last_upload', str(now)) success.append('Ceph report sent to {0}'.format(self.url)) self.log.info('Sent report to {0}'.format(self.url)) elif e == self.EndPoint.device: if 'device' in self.get_active_channels(): devices = self.gather_device_report() assert devices num_devs = 0 num_hosts = 0 for host, ls in devices.items(): self.log.debug('host %s devices %s' % (host, ls)) if not len(ls): continue fail_reason = self._try_post('devices', self.device_url, ls) if fail_reason: failed.append(fail_reason) else: num_devs += len(ls) num_hosts += 1 if num_devs: success.append('Reported %d devices across %d hosts' % (num_devs, len(devices))) if failed: return 1, '', '\n'.join(success + failed) return 0, '', '\n'.join(success) @CLIReadCommand('telemetry status') def status(self) -> Tuple[int, str, str]: ''' Show current configuration ''' r = {} for opt in self.MODULE_OPTIONS: r[opt['name']] = getattr(self, opt['name']) r['last_upload'] = (time.ctime(self.last_upload) if self.last_upload else self.last_upload) return 0, json.dumps(r, indent=4, sort_keys=True), '' @CLICommand('telemetry on') def on(self, license: Optional[str] = None) -> Tuple[int, str, str]: ''' Enable telemetry reports from this cluster ''' if license != LICENSE: return -errno.EPERM, '', f'''Telemetry data is licensed under the {LICENSE_NAME} ({LICENSE_URL}). To enable, add '--license {LICENSE}' to the 'ceph telemetry on' command.''' else: self.set_module_option('enabled', True) self.set_module_option('last_opt_revision', REVISION) return 0, '', '' @CLICommand('telemetry off') def off(self) -> Tuple[int, str, str]: ''' Disable telemetry reports from this cluster ''' self.set_module_option('enabled', False) self.set_module_option('last_opt_revision', 1) return 0, '', '' @CLICommand('telemetry send') def do_send(self, endpoint: Optional[List[EndPoint]] = None, license: Optional[str] = None) -> Tuple[int, str, str]: if self.last_opt_revision < LAST_REVISION_RE_OPT_IN and license != LICENSE: self.log.debug(('A telemetry send attempt while opted-out. ' 'Asking for license agreement')) return -errno.EPERM, '', f'''Telemetry data is licensed under the {LICENSE_NAME} ({LICENSE_URL}). To manually send telemetry data, add '--license {LICENSE}' to the 'ceph telemetry send' command. Please consider enabling the telemetry module with 'ceph telemetry on'.''' else: self.last_report = self.compile_report() return self.send(self.last_report, endpoint) @CLIReadCommand('telemetry show') def show(self, channels: Optional[List[str]] = None) -> Tuple[int, str, str]: ''' Show report of all channels ''' report = self.get_report(channels=channels) report = json.dumps(report, indent=4, sort_keys=True) if self.channel_device: report += ''' Device report is generated separately. To see it run 'ceph telemetry show-device'.''' return 0, report, '' @CLIReadCommand('telemetry show-device') def show_device(self) -> Tuple[int, str, str]: return 0, json.dumps(self.get_report('device'), indent=4, sort_keys=True), '' @CLIReadCommand('telemetry show-all') def show_all(self) -> Tuple[int, str, str]: return 0, json.dumps(self.get_report('all'), indent=4, sort_keys=True), '' def get_report(self, report_type: str = 'default', channels: Optional[List[str]] = None) -> Dict[str, Any]: if report_type == 'default': return self.compile_report(channels=channels) elif report_type == 'device': return self.gather_device_report() elif report_type == 'all': return { 'report': self.compile_report(channels=channels), 'device_report': self.gather_device_report() } return {} def self_test(self) -> None: report = self.compile_report() if len(report) == 0: raise RuntimeError('Report is empty') if 'report_id' not in report: raise RuntimeError('report_id not found in report') def shutdown(self) -> None: self.run = False self.event.set() def refresh_health_checks(self) -> None: health_checks = {} if self.enabled and self.last_opt_revision < LAST_REVISION_RE_OPT_IN: health_checks['TELEMETRY_CHANGED'] = { 'severity': 'warning', 'summary': 'Telemetry requires re-opt-in', 'detail': [ 'telemetry report includes new information; must re-opt-in (or out)' ] } self.set_health_checks(health_checks) def serve(self) -> None: self.load() self.config_notify() self.run = True self.log.debug('Waiting for mgr to warm up') self.event.wait(10) while self.run: self.event.clear() self.refresh_health_checks() if self.last_opt_revision < LAST_REVISION_RE_OPT_IN: self.log.debug('Not sending report until user re-opts-in') self.event.wait(1800) continue if not self.enabled: self.log.debug('Not sending report until configured to do so') self.event.wait(1800) continue now = int(time.time()) if not self.last_upload or \ (now - self.last_upload) > self.interval * 3600: self.log.info('Compiling and sending report to %s', self.url) try: self.last_report = self.compile_report() except Exception: self.log.exception('Exception while compiling report:') self.send(self.last_report) else: self.log.debug( 'Interval for sending new report has not expired') sleep = 3600 self.log.debug('Sleeping for %d seconds', sleep) self.event.wait(sleep) @staticmethod def can_run() -> Tuple[bool, str]: return True, ''
class Module(MgrModule): MODULE_OPTIONS = [ Option(name='hostname', default=None, desc='InfluxDB server hostname'), Option(name='port', type='int', default=8086, desc='InfluxDB server port'), Option(name='database', default='ceph', desc=('InfluxDB database name. You will need to create this ' 'database and grant write privileges to the configured ' 'username or the username must have admin privileges to ' 'create it.')), Option(name='username', default=None, desc='username of InfluxDB server user'), Option(name='password', default=None, desc='password of InfluxDB server user'), Option(name='interval', type='secs', min=5, default=30, desc='Time between reports to InfluxDB. Default 30 seconds.'), Option( name='ssl', default='false', desc= 'Use https connection for InfluxDB server. Use "true" or "false".' ), Option( name='verify_ssl', default='true', desc='Verify https cert for InfluxDB server. Use "true" or "false".' ), Option( name='threads', type='int', min=1, max=32, default=5, desc= 'How many worker threads should be spawned for sending data to InfluxDB.' ), Option( name='batch_size', type='int', default=5000, desc= 'How big batches of data points should be when sending to InfluxDB.' ), ] @property def config_keys(self) -> Dict[str, OptionValue]: return dict( (o['name'], o.get('default', None)) for o in self.MODULE_OPTIONS) COMMANDS = [{ "cmd": "influx config-set name=key,type=CephString " "name=value,type=CephString", "desc": "Set a configuration value", "perm": "rw" }, { "cmd": "influx config-show", "desc": "Show current configuration", "perm": "r" }, { "cmd": "influx send", "desc": "Force sending data to Influx", "perm": "rw" }] def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) self.event = Event() self.run = True self.config: Dict[str, OptionValue] = dict() self.workers: List[Thread] = list() self.queue: 'queue.Queue[Optional[List[Dict[str, str]]]]' = queue.Queue( maxsize=100) self.health_checks: Dict[str, Dict[str, Any]] = dict() def get_fsid(self) -> str: return self.get('mon_map')['fsid'] @staticmethod def can_run() -> Tuple[bool, str]: if InfluxDBClient is not None: return True, "" else: return False, "influxdb python module not found" @staticmethod def get_timestamp() -> str: return datetime.utcnow().isoformat() + 'Z' @staticmethod def chunk(l: Iterator[Dict[str, str]], n: int) -> Iterator[List[Dict[str, str]]]: try: while True: xs = [] for _ in range(n): xs.append(next(l)) yield xs except StopIteration: yield xs def queue_worker(self) -> None: while True: try: points = self.queue.get() if not points: self.log.debug('Worker shutting down') break start = time.time() with self.get_influx_client() as client: client.write_points(points, time_precision='ms') runtime = time.time() - start self.log.debug('Writing points %d to Influx took %.3f seconds', len(points), runtime) except RequestException as e: hostname = self.config['hostname'] port = self.config['port'] self.log.exception( f"Failed to connect to Influx host {hostname}:{port}") self.health_checks.update({ 'MGR_INFLUX_SEND_FAILED': { 'severity': 'warning', 'summary': 'Failed to send data to InfluxDB server ' f'at {hostname}:{port} due to an connection error', 'detail': [str(e)] } }) except InfluxDBClientError as e: self.health_checks.update({ 'MGR_INFLUX_SEND_FAILED': { 'severity': 'warning', 'summary': 'Failed to send data to InfluxDB', 'detail': [str(e)] } }) self.log.exception('Failed to send data to InfluxDB') except queue.Empty: continue except: self.log.exception( 'Unhandled Exception while sending to Influx') finally: self.queue.task_done() def get_latest(self, daemon_type: str, daemon_name: str, stat: str) -> int: data = self.get_counter(daemon_type, daemon_name, stat)[stat] if data: return data[-1][1] return 0 def get_df_stats(self, now) -> Tuple[List[Dict[str, Any]], Dict[str, str]]: df = self.get("df") data = [] pool_info = {} df_types = [ 'stored', 'kb_used', 'dirty', 'rd', 'rd_bytes', 'stored_raw', 'wr', 'wr_bytes', 'objects', 'max_avail', 'quota_objects', 'quota_bytes' ] for df_type in df_types: for pool in df['pools']: point = { "measurement": "ceph_pool_stats", "tags": { "pool_name": pool['name'], "pool_id": pool['id'], "type_instance": df_type, "fsid": self.get_fsid() }, "time": now, "fields": { "value": pool['stats'][df_type], } } data.append(point) pool_info.update({str(pool['id']): pool['name']}) return data, pool_info def get_pg_summary_osd(self, pool_info: Dict[str, str], now: str) -> Iterator[Dict[str, Any]]: pg_sum = self.get('pg_summary') osd_sum = pg_sum['by_osd'] for osd_id, stats in osd_sum.items(): metadata = self.get_metadata('osd', "%s" % osd_id) if not metadata: continue for stat in stats: yield { "measurement": "ceph_pg_summary_osd", "tags": { "ceph_daemon": "osd." + str(osd_id), "type_instance": stat, "host": metadata['hostname'] }, "time": now, "fields": { "value": stats[stat] } } def get_pg_summary_pool(self, pool_info: Dict[str, str], now: str) -> Iterator[Dict[str, Any]]: pool_sum = self.get('pg_summary')['by_pool'] for pool_id, stats in pool_sum.items(): try: pool_name = pool_info[pool_id] except KeyError: self.log.error( 'Unable to find pool name for pool {}'.format(pool_id)) continue for stat in stats: yield { "measurement": "ceph_pg_summary_pool", "tags": { "pool_name": pool_name, "pool_id": pool_id, "type_instance": stat, }, "time": now, "fields": { "value": stats[stat], } } def get_daemon_stats(self, now: str) -> Iterator[Dict[str, Any]]: for daemon, counters in self.get_all_perf_counters().items(): svc_type, svc_id = daemon.split(".", 1) metadata = self.get_metadata(svc_type, svc_id) if metadata is not None: hostname = metadata['hostname'] else: hostname = 'N/A' for path, counter_info in counters.items(): if counter_info['type'] & self.PERFCOUNTER_HISTOGRAM: continue value = counter_info['value'] yield { "measurement": "ceph_daemon_stats", "tags": { "ceph_daemon": daemon, "type_instance": path, "host": hostname, "fsid": self.get_fsid() }, "time": now, "fields": { "value": value } } def init_module_config(self) -> None: self.config['hostname'] = \ self.get_module_option("hostname", default=self.config_keys['hostname']) self.config['port'] = \ cast(int, self.get_module_option("port", default=self.config_keys['port'])) self.config['database'] = \ self.get_module_option("database", default=self.config_keys['database']) self.config['username'] = \ self.get_module_option("username", default=self.config_keys['username']) self.config['password'] = \ self.get_module_option("password", default=self.config_keys['password']) self.config['interval'] = \ cast(int, self.get_module_option("interval", default=self.config_keys['interval'])) self.config['threads'] = \ cast(int, self.get_module_option("threads", default=self.config_keys['threads'])) self.config['batch_size'] = \ cast(int, self.get_module_option("batch_size", default=self.config_keys['batch_size'])) ssl = cast( str, self.get_module_option("ssl", default=self.config_keys['ssl'])) self.config['ssl'] = ssl.lower() == 'true' verify_ssl = \ cast(str, self.get_module_option("verify_ssl", default=self.config_keys['verify_ssl'])) self.config['verify_ssl'] = verify_ssl.lower() == 'true' def gather_statistics(self) -> Iterator[Dict[str, str]]: now = self.get_timestamp() df_stats, pools = self.get_df_stats(now) return chain(df_stats, self.get_daemon_stats(now), self.get_pg_summary_osd(pools, now), self.get_pg_summary_pool(pools, now)) @contextmanager def get_influx_client(self) -> Iterator['InfluxDBClient']: client = InfluxDBClient(self.config['hostname'], self.config['port'], self.config['username'], self.config['password'], self.config['database'], self.config['ssl'], self.config['verify_ssl']) try: yield client finally: try: client.close() except AttributeError: # influxdb older than v5.0.0 pass def send_to_influx(self) -> bool: if not self.config['hostname']: self.log.error( "No Influx server configured, please set one using: " "ceph influx config-set hostname <hostname>") self.set_health_checks({ 'MGR_INFLUX_NO_SERVER': { 'severity': 'warning', 'summary': 'No InfluxDB server configured', 'detail': ['Configuration option hostname not set'] } }) return False self.health_checks = dict() self.log.debug("Sending data to Influx host: %s", self.config['hostname']) try: with self.get_influx_client() as client: databases = client.get_list_database() if {'name': self.config['database']} not in databases: self.log.info( "Database '%s' not found, trying to create " "(requires admin privs). You can also create " "manually and grant write privs to user " "'%s'", self.config['database'], self.config['database']) client.create_database(self.config['database']) client.create_retention_policy( name='8_weeks', duration='8w', replication='1', default=True, database=self.config['database']) self.log.debug('Gathering statistics') points = self.gather_statistics() for chunk in self.chunk(points, cast(int, self.config['batch_size'])): self.queue.put(chunk, block=False) self.log.debug('Queue currently contains %d items', self.queue.qsize()) return True except queue.Full: self.health_checks.update({ 'MGR_INFLUX_QUEUE_FULL': { 'severity': 'warning', 'summary': 'Failed to chunk to InfluxDB Queue', 'detail': [ 'Queue is full. InfluxDB might be slow with ' 'processing data' ] } }) self.log.error('Queue is full, failed to add chunk') return False except (RequestException, InfluxDBClientError) as e: self.health_checks.update({ 'MGR_INFLUX_DB_LIST_FAILED': { 'severity': 'warning', 'summary': 'Failed to list/create InfluxDB database', 'detail': [str(e)] } }) self.log.exception('Failed to list/create InfluxDB database') return False finally: self.set_health_checks(self.health_checks) def shutdown(self) -> None: self.log.info('Stopping influx module') self.run = False self.event.set() self.log.debug('Shutting down queue workers') for _ in self.workers: self.queue.put([]) self.queue.join() for worker in self.workers: worker.join() def self_test(self) -> Optional[str]: now = self.get_timestamp() daemon_stats = list(self.get_daemon_stats(now)) assert len(daemon_stats) df_stats, pools = self.get_df_stats(now) result = {'daemon_stats': daemon_stats, 'df_stats': df_stats} return json.dumps(result, indent=2, sort_keys=True) @CLIReadCommand('influx config-show') def config_show(self) -> Tuple[int, str, str]: """ Show current configuration """ return 0, json.dumps(self.config, sort_keys=True), '' @CLIWriteCommand('influx config-set') def config_set(self, key: str, value: str) -> Tuple[int, str, str]: if not value: return -errno.EINVAL, '', 'Value should not be empty' self.log.debug('Setting configuration option %s to %s', key, value) try: self.set_module_option(key, value) self.config[key] = self.get_module_option(key) return 0, 'Configuration option {0} updated'.format(key), '' except ValueError as e: return -errno.EINVAL, '', str(e) @CLICommand('influx send') def send(self) -> Tuple[int, str, str]: """ Force sending data to Influx """ self.send_to_influx() return 0, 'Sending data to Influx', '' def serve(self) -> None: if InfluxDBClient is None: self.log.error("Cannot transmit statistics: influxdb python " "module not found. Did you install it?") return self.log.info('Starting influx module') self.init_module_config() self.run = True self.log.debug('Starting %d queue worker threads', self.config['threads']) for i in range(cast(int, self.config['threads'])): worker = Thread(target=self.queue_worker, args=()) worker.setDaemon(True) worker.start() self.workers.append(worker) while self.run: start = time.time() self.send_to_influx() runtime = time.time() - start self.log.debug('Finished sending data to Influx in %.3f seconds', runtime) self.log.debug("Sleeping for %d seconds", self.config['interval']) self.event.wait(cast(float, self.config['interval']))
class Module(MgrModule, CherryPyConfig): """ dashboard module entrypoint """ COMMANDS = [ { 'cmd': 'dashboard set-jwt-token-ttl ' 'name=seconds,type=CephInt', 'desc': 'Set the JWT token TTL in seconds', 'perm': 'w' }, { 'cmd': 'dashboard get-jwt-token-ttl', 'desc': 'Get the JWT token TTL in seconds', 'perm': 'r' }, { "cmd": "dashboard create-self-signed-cert", "desc": "Create self signed certificate", "perm": "w" }, { "cmd": "dashboard grafana dashboards update", "desc": "Push dashboards to Grafana", "perm": "w", }, ] COMMANDS.extend(options_command_list()) COMMANDS.extend(SSO_COMMANDS) PLUGIN_MANAGER.hook.register_commands() MODULE_OPTIONS = [ Option(name='server_addr', type='str', default=get_default_addr()), Option(name='server_port', type='int', default=8080), Option(name='ssl_server_port', type='int', default=8443), Option(name='jwt_token_ttl', type='int', default=28800), Option(name='url_prefix', type='str', default=''), Option(name='key_file', type='str', default=''), Option(name='crt_file', type='str', default=''), Option(name='ssl', type='bool', default=True), Option(name='standby_behaviour', type='str', default='redirect', enum_allowed=['redirect', 'error']), Option(name='standby_error_status_code', type='int', default=500, min=400, max=599) ] MODULE_OPTIONS.extend(options_schema_list()) for options in PLUGIN_MANAGER.hook.get_options() or []: MODULE_OPTIONS.extend(options) __pool_stats = collections.defaultdict(lambda: collections.defaultdict( lambda: collections.deque(maxlen=10))) # type: dict def __init__(self, *args, **kwargs): super(Module, self).__init__(*args, **kwargs) CherryPyConfig.__init__(self) mgr.init(self) self._stopping = threading.Event() self.shutdown_event = threading.Event() self.ACCESS_CTRL_DB = None self.SSO_DB = None self.health_checks = {} @classmethod def can_run(cls): if cherrypy is None: return False, "Missing dependency: cherrypy" if not os.path.exists(cls.get_frontend_path()): return False, ( "Frontend assets not found at '{}': incomplete build?".format( cls.get_frontend_path())) return True, "" @classmethod def get_frontend_path(cls): current_dir = os.path.dirname(os.path.abspath(__file__)) path = os.path.join(current_dir, 'frontend/dist') if os.path.exists(path): return path else: path = os.path.join(current_dir, '../../../../build', 'src/pybind/mgr/dashboard', 'frontend/dist') return os.path.abspath(path) def serve(self): if 'COVERAGE_ENABLED' in os.environ: import coverage __cov = coverage.Coverage(config_file="{}/.coveragerc".format( os.path.dirname(__file__)), data_suffix=True) __cov.start() cherrypy.engine.subscribe('after_request', __cov.save) cherrypy.engine.subscribe('stop', __cov.stop) AuthManager.initialize() load_sso_db() uri = self.await_configuration() if uri is None: # We were shut down while waiting return # Publish the URI that others may use to access the service we're # about to start serving self.set_uri(uri) mapper, parent_urls = generate_routes(self.url_prefix) config = {} for purl in parent_urls: config[purl] = {'request.dispatch': mapper} cherrypy.tree.mount(None, config=config) PLUGIN_MANAGER.hook.setup() cherrypy.engine.start() NotificationQueue.start_queue() TaskManager.init() logger.info('Engine started.') update_dashboards = str_to_bool( self.get_module_option('GRAFANA_UPDATE_DASHBOARDS', 'False')) if update_dashboards: logger.info('Starting Grafana dashboard task') TaskManager.run( 'grafana/dashboards/update', {}, push_local_dashboards, kwargs=dict(tries=10, sleep=60), ) # wait for the shutdown event self.shutdown_event.wait() self.shutdown_event.clear() NotificationQueue.stop() cherrypy.engine.stop() logger.info('Engine stopped') def shutdown(self): super(Module, self).shutdown() CherryPyConfig.shutdown(self) logger.info('Stopping engine...') self.shutdown_event.set() def _set_ssl_item(self, item_label: str, item_key: 'SslConfigKey' = 'crt', mgr_id: Optional[str] = None, inbuf: Optional[str] = None): if inbuf is None: return -errno.EINVAL, '', f'Please specify the {item_label} with "-i" option' if mgr_id is not None: self.set_store(_get_localized_key(mgr_id, item_key), inbuf) else: self.set_store(item_key, inbuf) return 0, f'SSL {item_label} updated', '' @CLIWriteCommand("dashboard set-ssl-certificate") def set_ssl_certificate(self, mgr_id: Optional[str] = None, inbuf: Optional[str] = None): return self._set_ssl_item('certificate', 'crt', mgr_id, inbuf) @CLIWriteCommand("dashboard set-ssl-certificate-key") def set_ssl_certificate_key(self, mgr_id: Optional[str] = None, inbuf: Optional[str] = None): return self._set_ssl_item('certificate key', 'key', mgr_id, inbuf) @CLIWriteCommand("dashboard create-self-signed-cert") def set_mgr_created_self_signed_cert(self): cert, pkey = create_self_signed_cert('IT', 'ceph-dashboard') result = HandleCommandResult(*self.set_ssl_certificate(inbuf=cert)) if result.retval != 0: return result result = HandleCommandResult(*self.set_ssl_certificate_key(inbuf=pkey)) if result.retval != 0: return result return 0, 'Self-signed certificate created', '' @CLICommand("dashboard get issue") def get_issues_cli(self, issue_number: int): try: issue_number = int(issue_number) except TypeError: return -errno.EINVAL, '', f'Invalid issue number {issue_number}' tracker_client = CephTrackerClient() try: response = tracker_client.get_issues(issue_number) except RequestException as error: if error.status_code == 404: return -errno.EINVAL, '', f'Issue {issue_number} not found' else: return -errno.EREMOTEIO, '', f'Error: {str(error)}' return 0, str(response), '' @CLICommand("dashboard create issue") def report_issues_cli(self, project: str, tracker: str, subject: str, description: str): ''' Create an issue in the Ceph Issue tracker Syntax: ceph dashboard create issue <project> <bug|feature> <subject> <description> ''' try: feedback = Feedback(Feedback.Project[project].value, Feedback.TrackerType[tracker].value, subject, description) except KeyError: return -errno.EINVAL, '', 'Invalid arguments' tracker_client = CephTrackerClient() try: response = tracker_client.create_issue(feedback) except RequestException as error: if error.status_code == 401: return -errno.EINVAL, '', 'Invalid API Key' else: return -errno.EINVAL, '', f'Error: {str(error)}' except Exception: return -errno.EINVAL, '', 'Ceph Tracker API key not set' return 0, str(response), '' @CLIWriteCommand("dashboard set-rgw-credentials") def set_rgw_credentials(self): try: configure_rgw_credentials() except Exception as error: return -errno.EINVAL, '', str(error) return 0, 'RGW credentials configured', '' def handle_command(self, inbuf, cmd): # pylint: disable=too-many-return-statements res = handle_option_command(cmd, inbuf) if res[0] != -errno.ENOSYS: return res res = handle_sso_command(cmd) if res[0] != -errno.ENOSYS: return res if cmd['prefix'] == 'dashboard set-jwt-token-ttl': self.set_module_option('jwt_token_ttl', str(cmd['seconds'])) return 0, 'JWT token TTL updated', '' if cmd['prefix'] == 'dashboard get-jwt-token-ttl': ttl = self.get_module_option('jwt_token_ttl', JwtManager.JWT_TOKEN_TTL) return 0, str(ttl), '' if cmd['prefix'] == 'dashboard grafana dashboards update': push_local_dashboards() return 0, 'Grafana dashboards updated', '' return (-errno.EINVAL, '', 'Command not found \'{0}\''.format(cmd['prefix'])) def notify(self, notify_type, notify_id): NotificationQueue.new_notification(notify_type, notify_id) def get_updated_pool_stats(self): df = self.get('df') pool_stats = {p['id']: p['stats'] for p in df['pools']} now = time.time() for pool_id, stats in pool_stats.items(): for stat_name, stat_val in stats.items(): self.__pool_stats[pool_id][stat_name].append((now, stat_val)) return self.__pool_stats def config_notify(self): """ This method is called whenever one of our config options is changed. """ PLUGIN_MANAGER.hook.config_notify() def refresh_health_checks(self): self.set_health_checks(self.health_checks)
class Module(MgrModule): MODULE_OPTIONS = [ Option( name='enable_monitoring', default=True, type='bool', desc='monitor device health metrics', runtime=True, ), Option( name='scrape_frequency', default=86400, type='secs', desc='how frequently to scrape device health metrics', runtime=True, ), Option( name='pool_name', default='device_health_metrics', type='str', desc='name of pool in which to store device health metrics', runtime=True, ), Option( name='retention_period', default=(86400 * 180), type='secs', desc='how long to retain device health metrics', runtime=True, ), Option( name='mark_out_threshold', default=(86400 * 14 * 2), type='secs', desc='automatically mark OSD if it may fail before this long', runtime=True, ), Option( name='warn_threshold', default=(86400 * 14 * 6), type='secs', desc='raise health warning if OSD may fail before this long', runtime=True, ), Option( name='self_heal', default=True, type='bool', desc='preemptively heal cluster around devices that may fail', runtime=True, ), Option( name='sleep_interval', default=600, type='secs', desc='how frequently to wake up and check device health', runtime=True, ), ] def __init__(self, *args, **kwargs): super(Module, self).__init__(*args, **kwargs) # populate options (just until serve() runs) for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], opt['default']) # other self.run = True self.event = Event() self.has_device_pool = False def is_valid_daemon_name(self, who): parts = who.split('.') if len(parts) != 2: return False return parts[0] in ('osd', 'mon') def handle_command(self, _, cmd): self.log.error("handle_command") @CLICommand('device query-daemon-health-metrics', perm='r') def do_query_daemon_health_metrics(self, who: str): ''' Get device health metrics for a given daemon ''' if not self.is_valid_daemon_name(who): return -errno.EINVAL, '', 'not a valid mon or osd daemon name' (daemon_type, daemon_id) = who.split('.') result = CommandResult('') self.send_command(result, daemon_type, daemon_id, json.dumps({ 'prefix': 'smart', 'format': 'json', }), '') return result.wait() @CLICommand('device scrape-daemon-health-metrics', perm='r') def do_scrape_daemon_health_metrics(self, who: str): ''' Scrape and store device health metrics for a given daemon ''' if not self.is_valid_daemon_name(who): return -errno.EINVAL, '', 'not a valid mon or osd daemon name' (daemon_type, daemon_id) = who.split('.') return self.scrape_daemon(daemon_type, daemon_id) @CLICommand('device scrape-daemon-health-metrics', perm='r') def do_scrape_health_metrics(self, devid: Optional[str] = None): ''' Scrape and store device health metrics ''' if devid is None: return self.scrape_all() else: return self.scrape_device(devid) @CLICommand('device get-health-metrics', perm='r') def do_get_health_metrics(self, devid: str, sample: Optional[str] = None): ''' Show stored device metrics for the device ''' return self.show_device_metrics(devid, sample) @CLICommand('device check-health', perm='rw') def do_check_health(self): ''' Check life expectancy of devices ''' return self.check_health() @CLICommand('device monitoring on', perm='rw') def do_monitoring_on(self): ''' Enable device health monitoring ''' self.set_module_option('enable_monitoring', True) self.event.set() @CLICommand('device monitoring off', perm='rw') def do_monitoring_off(self): ''' Disable device health monitoring ''' self.set_module_option('enable_monitoring', False) self.set_health_checks({}) # avoid stuck health alerts @CLICommand('device predict-life-expectancy', perm='r') def do_predict_life_expectancy(self, devid: str): ''' Predict life expectancy with local predictor ''' return self.predict_lift_expectancy(devid) def self_test(self): self.config_notify() osdmap = self.get('osd_map') osd_id = osdmap['osds'][0]['osd'] osdmeta = self.get('osd_metadata') devs = osdmeta.get(str(osd_id), {}).get('device_ids') if devs: devid = devs.split()[0].split('=')[1] (r, before, err) = self.show_device_metrics(devid, '') assert r == 0 (r, out, err) = self.scrape_device(devid) assert r == 0 (r, after, err) = self.show_device_metrics(devid, '') assert r == 0 assert before != after def config_notify(self): for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name'])) def notify(self, notify_type, notify_id): # create device_health_metrics pool if it doesn't exist if notify_type == "osd_map" and self.enable_monitoring: if not self.has_device_pool: self.create_device_pool() self.has_device_pool = True def create_device_pool(self): self.log.debug('create %s pool' % self.pool_name) # create pool result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd pool create', 'format': 'json', 'pool': self.pool_name, 'pg_num': 1, 'pg_num_min': 1, }), '') r, outb, outs = result.wait() assert r == 0 # set pool application result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd pool application enable', 'format': 'json', 'pool': self.pool_name, 'app': 'mgr_devicehealth', }), '') r, outb, outs = result.wait() assert r == 0 def serve(self): self.log.info("Starting") self.config_notify() last_scrape = None ls = self.get_store('last_scrape') if ls: try: last_scrape = datetime.strptime(ls, TIME_FORMAT) except ValueError: pass self.log.debug('Last scrape %s', last_scrape) while self.run: if self.enable_monitoring: self.log.debug('Running') self.check_health() now = datetime.utcnow() if not last_scrape: next_scrape = now else: # align to scrape interval scrape_frequency = self.scrape_frequency or 86400 seconds = (last_scrape - datetime.utcfromtimestamp(0)).total_seconds() seconds -= seconds % scrape_frequency seconds += scrape_frequency next_scrape = datetime.utcfromtimestamp(seconds) if last_scrape: self.log.debug('Last scrape %s, next scrape due %s', last_scrape.strftime(TIME_FORMAT), next_scrape.strftime(TIME_FORMAT)) else: self.log.debug('Last scrape never, next scrape due %s', next_scrape.strftime(TIME_FORMAT)) if now >= next_scrape: self.scrape_all() self.predict_all_devices() last_scrape = now self.set_store('last_scrape', last_scrape.strftime(TIME_FORMAT)) # sleep sleep_interval = self.sleep_interval or 60 self.log.debug('Sleeping for %d seconds', sleep_interval) ret = self.event.wait(sleep_interval) self.event.clear() def shutdown(self): self.log.info('Stopping') self.run = False self.event.set() def open_connection(self, create_if_missing=True): osdmap = self.get("osd_map") assert osdmap is not None if len(osdmap['osds']) == 0: return None if not self.has_device_pool: if not create_if_missing: return None if self.enable_monitoring: self.create_device_pool() self.has_device_pool = True ioctx = self.rados.open_ioctx(self.pool_name) return ioctx def scrape_daemon(self, daemon_type, daemon_id): ioctx = self.open_connection() if not ioctx: return 0, "", "" raw_smart_data = self.do_scrape_daemon(daemon_type, daemon_id) if raw_smart_data: for device, raw_data in raw_smart_data.items(): data = self.extract_smart_features(raw_data) if device and data: self.put_device_metrics(ioctx, device, data) ioctx.close() return 0, "", "" def scrape_all(self): osdmap = self.get("osd_map") assert osdmap is not None ioctx = self.open_connection() if not ioctx: return 0, "", "" did_device = {} ids = [] for osd in osdmap['osds']: ids.append(('osd', str(osd['osd']))) monmap = self.get("mon_map") for mon in monmap['mons']: ids.append(('mon', mon['name'])) for daemon_type, daemon_id in ids: raw_smart_data = self.do_scrape_daemon(daemon_type, daemon_id) if not raw_smart_data: continue for device, raw_data in raw_smart_data.items(): if device in did_device: self.log.debug('skipping duplicate %s' % device) continue did_device[device] = 1 data = self.extract_smart_features(raw_data) if device and data: self.put_device_metrics(ioctx, device, data) ioctx.close() return 0, "", "" def scrape_device(self, devid): r = self.get("device " + devid) if not r or 'device' not in r.keys(): return -errno.ENOENT, '', 'device ' + devid + ' not found' daemons = r['device'].get('daemons', []) if not daemons: return (-errno.EAGAIN, '', 'device ' + devid + ' not claimed by any active daemons') (daemon_type, daemon_id) = daemons[0].split('.') ioctx = self.open_connection() if not ioctx: return 0, "", "" raw_smart_data = self.do_scrape_daemon(daemon_type, daemon_id, devid=devid) if raw_smart_data: for device, raw_data in raw_smart_data.items(): data = self.extract_smart_features(raw_data) if device and data: self.put_device_metrics(ioctx, device, data) ioctx.close() return 0, "", "" def do_scrape_daemon(self, daemon_type, daemon_id, devid=''): """ :return: a dict, or None if the scrape failed. """ self.log.debug('do_scrape_daemon %s.%s' % (daemon_type, daemon_id)) result = CommandResult('') self.send_command( result, daemon_type, daemon_id, json.dumps({ 'prefix': 'smart', 'format': 'json', 'devid': devid, }), '') r, outb, outs = result.wait() try: return json.loads(outb) except (IndexError, ValueError): self.log.error( "Fail to parse JSON result from daemon {0}.{1} ({2})".format( daemon_type, daemon_id, outb)) def put_device_metrics(self, ioctx, devid, data): assert devid old_key = datetime.utcnow() - timedelta(seconds=self.retention_period) prune = old_key.strftime(TIME_FORMAT) self.log.debug('put_device_metrics device %s prune %s' % (devid, prune)) erase = [] try: with rados.ReadOpCtx() as op: # FIXME omap_iter, ret = ioctx.get_omap_keys(op, "", MAX_SAMPLES) assert ret == 0 ioctx.operate_read_op(op, devid) for key, _ in list(omap_iter): if key >= prune: break erase.append(key) except rados.ObjectNotFound: # The object doesn't already exist, no problem. pass except rados.Error as e: # Do not proceed with writes if something unexpected # went wrong with the reads. self.log.exception("Error reading OMAP: {0}".format(e)) return key = datetime.utcnow().strftime(TIME_FORMAT) self.log.debug('put_device_metrics device %s key %s = %s, erase %s' % (devid, key, data, erase)) with rados.WriteOpCtx() as op: ioctx.set_omap(op, (key, ), (str(json.dumps(data)), )) if len(erase): ioctx.remove_omap_keys(op, tuple(erase)) ioctx.operate_write_op(op, devid) def _get_device_metrics(self, devid, sample=None, min_sample=None): res = {} ioctx = self.open_connection(create_if_missing=False) if not ioctx: return {} with ioctx: with rados.ReadOpCtx() as op: omap_iter, ret = ioctx.get_omap_vals(op, min_sample or '', sample or '', MAX_SAMPLES) # fixme assert ret == 0 try: ioctx.operate_read_op(op, devid) for key, value in list(omap_iter): if sample and key != sample: break if min_sample and key < min_sample: break try: v = json.loads(value) except (ValueError, IndexError): self.log.debug( 'unable to parse value for %s: "%s"' % (key, value)) pass res[key] = v except rados.ObjectNotFound: pass except rados.Error as e: self.log.exception( "RADOS error reading omap: {0}".format(e)) raise return res def show_device_metrics(self, devid, sample): # verify device exists r = self.get("device " + devid) if not r or 'device' not in r.keys(): return -errno.ENOENT, '', 'device ' + devid + ' not found' # fetch metrics res = self._get_device_metrics(devid, sample=sample) return 0, json.dumps(res, indent=4, sort_keys=True), '' def check_health(self): self.log.info('Check health') config = self.get('config') min_in_ratio = float(config.get('mon_osd_min_in_ratio')) mark_out_threshold_td = timedelta(seconds=self.mark_out_threshold) warn_threshold_td = timedelta(seconds=self.warn_threshold) checks = {} health_warnings = { DEVICE_HEALTH: [], DEVICE_HEALTH_IN_USE: [], } devs = self.get("devices") osds_in = {} osds_out = {} now = datetime.utcnow() osdmap = self.get("osd_map") assert osdmap is not None for dev in devs['devices']: devid = dev['devid'] if 'life_expectancy_max' not in dev: continue # ignore devices that are not consumed by any daemons if not dev['daemons']: continue if not dev['life_expectancy_max'] or \ dev['life_expectancy_max'] == '0.000000': continue # life_expectancy_(min/max) is in the format of: # '%Y-%m-%dT%H:%M:%S.%f%z', e.g.: # '2019-01-20T21:12:12.000000Z' life_expectancy_max = datetime.strptime(dev['life_expectancy_max'], '%Y-%m-%dT%H:%M:%S.%f%z') self.log.debug('device %s expectancy max %s', dev, life_expectancy_max) if life_expectancy_max - now <= mark_out_threshold_td: if self.self_heal: # dev['daemons'] == ["osd.0","osd.1","osd.2"] if dev['daemons']: osds = [ x for x in dev['daemons'] if x.startswith('osd.') ] osd_ids = map(lambda x: x[4:], osds) for _id in osd_ids: if self.is_osd_in(osdmap, _id): osds_in[_id] = life_expectancy_max else: osds_out[_id] = 1 if life_expectancy_max - now <= warn_threshold_td: # device can appear in more than one location in case # of SCSI multipath device_locations = map(lambda x: x['host'] + ':' + x['dev'], dev['location']) health_warnings[DEVICE_HEALTH].append( '%s (%s); daemons %s; life expectancy between %s and %s' % (dev['devid'], ','.join(device_locations), ','.join( dev.get('daemons', ['none'])), dev['life_expectancy_max'], dev.get('life_expectancy_max', 'unknown'))) # OSD might be marked 'out' (which means it has no # data), however PGs are still attached to it. for _id in osds_out: num_pgs = self.get_osd_num_pgs(_id) if num_pgs > 0: health_warnings[DEVICE_HEALTH_IN_USE].append( 'osd.%s is marked out ' 'but still has %s PG(s)' % (_id, num_pgs)) if osds_in: self.log.debug('osds_in %s' % osds_in) # calculate target in ratio num_osds = len(osdmap['osds']) num_in = len([x for x in osdmap['osds'] if x['in']]) num_bad = len(osds_in) # sort with next-to-fail first bad_osds = sorted(osds_in.items(), key=operator.itemgetter(1)) did = 0 to_mark_out = [] for osd_id, when in bad_osds: ratio = float(num_in - did - 1) / float(num_osds) if ratio < min_in_ratio: final_ratio = float(num_in - num_bad) / float(num_osds) checks[DEVICE_HEALTH_TOOMANY] = { 'severity': 'warning', 'summary': HEALTH_MESSAGES[DEVICE_HEALTH_TOOMANY], 'detail': [ '%d OSDs with failing device(s) would bring "in" ratio to %f < mon_osd_min_in_ratio %f' % (num_bad - did, final_ratio, min_in_ratio) ] } break to_mark_out.append(osd_id) did += 1 if to_mark_out: self.mark_out_etc(to_mark_out) for warning, ls in health_warnings.items(): n = len(ls) if n: checks[warning] = { 'severity': 'warning', 'summary': HEALTH_MESSAGES[warning] % n, 'count': len(ls), 'detail': ls, } self.set_health_checks(checks) return 0, "", "" def is_osd_in(self, osdmap, osd_id): for osd in osdmap['osds']: if str(osd_id) == str(osd['osd']): return bool(osd['in']) return False def get_osd_num_pgs(self, osd_id): stats = self.get('osd_stats') assert stats is not None for stat in stats['osd_stats']: if str(osd_id) == str(stat['osd']): return stat['num_pgs'] return -1 def mark_out_etc(self, osd_ids): self.log.info('Marking out OSDs: %s' % osd_ids) result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd out', 'format': 'json', 'ids': osd_ids, }), '') r, outb, outs = result.wait() if r != 0: self.log.warning( 'Could not mark OSD %s out. r: [%s], outb: [%s], outs: [%s]', osd_ids, r, outb, outs) for osd_id in osd_ids: result = CommandResult('') self.send_command( result, 'mon', '', json.dumps({ 'prefix': 'osd primary-affinity', 'format': 'json', 'id': int(osd_id), 'weight': 0.0, }), '') r, outb, outs = result.wait() if r != 0: self.log.warning( 'Could not set osd.%s primary-affinity, ' 'r: [%s], outb: [%s], outs: [%s]', osd_id, r, outb, outs) def extract_smart_features(self, raw): # FIXME: extract and normalize raw smartctl --json output and # generate a dict of the fields we care about. return raw def predict_lift_expectancy(self, devid): plugin_name = '' model = self.get_ceph_option('device_failure_prediction_mode') if model and model.lower() == 'local': plugin_name = 'diskprediction_local' else: return -1, '', 'unable to enable any disk prediction model[local/cloud]' try: can_run, _ = self.remote(plugin_name, 'can_run') if can_run: return self.remote(plugin_name, 'predict_life_expectancy', devid=devid) except: return -1, '', 'unable to invoke diskprediction local or remote plugin' def predict_all_devices(self): plugin_name = '' model = self.get_ceph_option('device_failure_prediction_mode') if model and model.lower() == 'local': plugin_name = 'diskprediction_local' else: return -1, '', 'unable to enable any disk prediction model[local/cloud]' try: can_run, _ = self.remote(plugin_name, 'can_run') if can_run: return self.remote(plugin_name, 'predict_all_devices') except: return -1, '', 'unable to invoke diskprediction local or remote plugin' def get_recent_device_metrics(self, devid, min_sample): return self._get_device_metrics(devid, min_sample=min_sample) def get_time_format(self): return TIME_FORMAT
class Module(MgrModule): MODULE_OPTIONS = [ Option(name='sleep_interval', default=600), Option(name='predict_interval', default=86400), Option(name='predictor_model', default='prophetstor') ] def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) # options for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], opt['default']) # other self._run = True self._event = Event() # for mypy which does not run the code if TYPE_CHECKING: self.sleep_interval = 0 self.predict_interval = 0 self.predictor_model = '' def config_notify(self) -> None: for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name'])) if self.get_ceph_option('device_failure_prediction_mode') == 'local': self._event.set() def refresh_config(self) -> None: for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name'])) def self_test(self) -> None: self.log.debug('self_test enter') ret, out, err = self.predict_all_devices() assert ret == 0 def serve(self) -> None: self.log.info('Starting diskprediction local module') self.config_notify() last_predicted = None ls = self.get_store('last_predicted') if ls: try: last_predicted = datetime.datetime.strptime(ls, TIME_FORMAT) except ValueError: pass self.log.debug('Last predicted %s', last_predicted) while self._run: self.refresh_config() mode = self.get_ceph_option('device_failure_prediction_mode') if mode == 'local': now = datetime.datetime.utcnow() if not last_predicted: next_predicted = now else: predicted_frequency = self.predict_interval or 86400 seconds = (last_predicted - datetime.datetime.utcfromtimestamp(0)).total_seconds() seconds -= seconds % predicted_frequency seconds += predicted_frequency next_predicted = datetime.datetime.utcfromtimestamp(seconds) self.log.debug('Last scrape %s, next scrape due %s', last_predicted.strftime(TIME_FORMAT), next_predicted.strftime(TIME_FORMAT)) if now >= next_predicted: self.predict_all_devices() last_predicted = now self.set_store('last_predicted', last_predicted.strftime(TIME_FORMAT)) sleep_interval = self.sleep_interval or 60 self.log.debug('Sleeping for %d seconds', sleep_interval) self._event.wait(sleep_interval) self._event.clear() def shutdown(self) -> None: self.log.info('Stopping') self._run = False self._event.set() @staticmethod def _convert_timestamp(predicted_timestamp: int, life_expectancy_day: int) -> str: """ :param predicted_timestamp: unit is nanoseconds :param life_expectancy_day: unit is seconds :return: date format '%Y-%m-%d' ex. 2018-01-01 """ return datetime.datetime.fromtimestamp( predicted_timestamp / (1000 ** 3) + life_expectancy_day).strftime('%Y-%m-%d') def _predict_life_expentancy(self, devid: str) -> str: predicted_result = '' health_data: Dict[str, Dict[str, Any]] = {} predict_datas: List[DevSmartT] = [] try: r, outb, outs = self.remote( 'devicehealth', 'show_device_metrics', devid=devid, sample='') if r != 0: self.log.error('failed to get device %s health', devid) health_data = {} else: health_data = json.loads(outb) except Exception as e: self.log.error('failed to get device %s health data due to %s', devid, str(e)) # initialize appropriate disk failure predictor model obj_predictor = Predictor.create(self.predictor_model) if obj_predictor is None: self.log.error('invalid value received for MODULE_OPTIONS.predictor_model') return predicted_result try: obj_predictor.initialize( "{}/models/{}".format(get_diskfailurepredictor_path(), self.predictor_model)) except Exception as e: self.log.error('Error initializing predictor: %s', e) return predicted_result if len(health_data) >= 6: o_keys = sorted(health_data.keys(), reverse=True) for o_key in o_keys: # get values for current day (?) dev_smart = {} s_val = health_data[o_key] # add all smart attributes ata_smart = s_val.get('ata_smart_attributes', {}) for attr in ata_smart.get('table', []): # get raw smart values if attr.get('raw', {}).get('string') is not None: if str(attr.get('raw', {}).get('string', '0')).isdigit(): dev_smart['smart_%s_raw' % attr.get('id')] = \ int(attr.get('raw', {}).get('string', '0')) else: if str(attr.get('raw', {}).get('string', '0')).split(' ')[0].isdigit(): dev_smart['smart_%s_raw' % attr.get('id')] = \ int(attr.get('raw', {}).get('string', '0').split(' ')[0]) else: dev_smart['smart_%s_raw' % attr.get('id')] = \ attr.get('raw', {}).get('value', 0) # get normalized smart values if attr.get('value') is not None: dev_smart['smart_%s_normalized' % attr.get('id')] = \ attr.get('value') # add power on hours manually if not available in smart attributes power_on_time = s_val.get('power_on_time', {}).get('hours') if power_on_time is not None: dev_smart['smart_9_raw'] = int(power_on_time) # add device capacity user_capacity = s_val.get('user_capacity', {}).get('bytes') if user_capacity is not None: dev_smart['user_capacity'] = user_capacity else: self.log.debug('user_capacity not found in smart attributes list') # add device model model_name = s_val.get('model_name') if model_name is not None: dev_smart['model_name'] = model_name # add vendor vendor = s_val.get('vendor') if vendor is not None: dev_smart['vendor'] = vendor # if smart data was found, then add that to list if dev_smart: predict_datas.append(dev_smart) if len(predict_datas) >= 12: break else: self.log.error('unable to predict device due to health data records less than 6 days') if len(predict_datas) >= 6: predicted_result = obj_predictor.predict(predict_datas) return predicted_result def predict_life_expectancy(self, devid: str) -> Tuple[int, str, str]: result = self._predict_life_expentancy(devid) if result.lower() == 'good': return 0, '>6w', '' elif result.lower() == 'warning': return 0, '>=2w and <=6w', '' elif result.lower() == 'bad': return 0, '<2w', '' else: return 0, 'unknown', '' def _reset_device_life_expectancy(self, device_id: str) -> int: result = CommandResult('') self.send_command(result, 'mon', '', json.dumps({ 'prefix': 'device rm-life-expectancy', 'devid': device_id }), '') ret, _, outs = result.wait() if ret != 0: self.log.error( 'failed to reset device life expectancy, %s' % outs) return ret def _set_device_life_expectancy(self, device_id: str, from_date: str, to_date: Optional[str] = None) -> int: result = CommandResult('') if to_date is None: self.send_command(result, 'mon', '', json.dumps({ 'prefix': 'device set-life-expectancy', 'devid': device_id, 'from': from_date }), '') else: self.send_command(result, 'mon', '', json.dumps({ 'prefix': 'device set-life-expectancy', 'devid': device_id, 'from': from_date, 'to': to_date }), '') ret, _, outs = result.wait() if ret != 0: self.log.error( 'failed to set device life expectancy, %s' % outs) return ret def predict_all_devices(self) -> Tuple[int, str, str]: self.log.debug('predict_all_devices') devices = self.get('devices').get('devices', []) for devInfo in devices: if not devInfo.get('daemons'): continue if not devInfo.get('devid'): continue self.log.debug('%s' % devInfo) result = self._predict_life_expentancy(devInfo['devid']) if result == 'unknown': self._reset_device_life_expectancy(devInfo['devid']) continue predicted = int(time.time() * (1000 ** 3)) if result.lower() == 'good': life_expectancy_day_min = (TIME_WEEK * 6) + TIME_DAYS life_expectancy_day_max = 0 elif result.lower() == 'warning': life_expectancy_day_min = (TIME_WEEK * 2) life_expectancy_day_max = (TIME_WEEK * 6) elif result.lower() == 'bad': life_expectancy_day_min = 0 life_expectancy_day_max = (TIME_WEEK * 2) - TIME_DAYS else: predicted = 0 life_expectancy_day_min = 0 life_expectancy_day_max = 0 if predicted and devInfo['devid'] and life_expectancy_day_min: from_date = None to_date = None try: assert life_expectancy_day_min from_date = self._convert_timestamp(predicted, life_expectancy_day_min) if life_expectancy_day_max: to_date = self._convert_timestamp(predicted, life_expectancy_day_max) self._set_device_life_expectancy(devInfo['devid'], from_date, to_date) self._logger.info( 'succeed to set device {} life expectancy from: {}, to: {}'.format( devInfo['devid'], from_date, to_date)) except Exception as e: self._logger.error( 'failed to set device {} life expectancy from: {}, to: {}, {}'.format( devInfo['devid'], from_date, to_date, str(e))) else: self._reset_device_life_expectancy(devInfo['devid']) return 0, 'succeed to predicted all devices', ''
class Module(orchestrator.OrchestratorClientMixin, MgrModule): COMMANDS = [ { 'cmd': 'fs volume ls', 'desc': "List volumes", 'perm': 'r' }, { 'cmd': 'fs volume create ' f'name=name,type=CephString,goodchars={goodchars} ' 'name=placement,type=CephString,req=false ', 'desc': "Create a CephFS volume", 'perm': 'rw' }, { 'cmd': 'fs volume rm ' 'name=vol_name,type=CephString ' 'name=yes-i-really-mean-it,type=CephString,req=false ', 'desc': "Delete a FS volume by passing --yes-i-really-mean-it flag", 'perm': 'rw' }, { 'cmd': 'fs subvolumegroup ls ' 'name=vol_name,type=CephString ', 'desc': "List subvolumegroups", 'perm': 'r' }, { 'cmd': 'fs subvolumegroup create ' 'name=vol_name,type=CephString ' f'name=group_name,type=CephString,goodchars={goodchars} ' 'name=pool_layout,type=CephString,req=false ' 'name=uid,type=CephInt,req=false ' 'name=gid,type=CephInt,req=false ' 'name=mode,type=CephString,req=false ', 'desc': "Create a CephFS subvolume group in a volume, and optionally, " "with a specific data pool layout, and a specific numeric mode", 'perm': 'rw' }, { 'cmd': 'fs subvolumegroup rm ' 'name=vol_name,type=CephString ' 'name=group_name,type=CephString ' 'name=force,type=CephBool,req=false ', 'desc': "Delete a CephFS subvolume group in a volume", 'perm': 'rw' }, { 'cmd': 'fs subvolume ls ' 'name=vol_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "List subvolumes", 'perm': 'r' }, { 'cmd': 'fs subvolume create ' 'name=vol_name,type=CephString ' f'name=sub_name,type=CephString,goodchars={goodchars} ' 'name=size,type=CephInt,req=false ' 'name=group_name,type=CephString,req=false ' 'name=pool_layout,type=CephString,req=false ' 'name=uid,type=CephInt,req=false ' 'name=gid,type=CephInt,req=false ' 'name=mode,type=CephString,req=false ' 'name=namespace_isolated,type=CephBool,req=false ', 'desc': "Create a CephFS subvolume in a volume, and optionally, " "with a specific size (in bytes), a specific data pool layout, " "a specific mode, in a specific subvolume group and in separate " "RADOS namespace", 'perm': 'rw' }, { 'cmd': 'fs subvolume rm ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=group_name,type=CephString,req=false ' 'name=force,type=CephBool,req=false ' 'name=retain_snapshots,type=CephBool,req=false ', 'desc': "Delete a CephFS subvolume in a volume, and optionally, " "in a specific subvolume group, force deleting a cancelled or failed " "clone, and retaining existing subvolume snapshots", 'perm': 'rw' }, { 'cmd': 'fs subvolume authorize ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=auth_id,type=CephString ' 'name=group_name,type=CephString,req=false ' 'name=access_level,type=CephString,req=false ' 'name=tenant_id,type=CephString,req=false ' 'name=allow_existing_id,type=CephBool,req=false ', 'desc': "Allow a cephx auth ID access to a subvolume", 'perm': 'rw' }, { 'cmd': 'fs subvolume deauthorize ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=auth_id,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "Deny a cephx auth ID access to a subvolume", 'perm': 'rw' }, { 'cmd': 'fs subvolume authorized_list ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "List auth IDs that have access to a subvolume", 'perm': 'r' }, { 'cmd': 'fs subvolume evict ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=auth_id,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "Evict clients based on auth IDs and subvolume mounted", 'perm': 'rw' }, { 'cmd': 'fs subvolumegroup getpath ' 'name=vol_name,type=CephString ' 'name=group_name,type=CephString ', 'desc': "Get the mountpath of a CephFS subvolume group in a volume", 'perm': 'r' }, { 'cmd': 'fs subvolume getpath ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "Get the mountpath of a CephFS subvolume in a volume, " "and optionally, in a specific subvolume group", 'perm': 'rw' }, { 'cmd': 'fs subvolume info ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "Get the metadata of a CephFS subvolume in a volume, " "and optionally, in a specific subvolume group", 'perm': 'r' }, { 'cmd': 'fs subvolumegroup pin' ' name=vol_name,type=CephString' ' name=group_name,type=CephString,req=true' ' name=pin_type,type=CephChoices,strings=export|distributed|random' ' name=pin_setting,type=CephString,req=true', 'desc': "Set MDS pinning policy for subvolumegroup", 'perm': 'rw' }, { 'cmd': 'fs subvolumegroup snapshot ls ' 'name=vol_name,type=CephString ' 'name=group_name,type=CephString ', 'desc': "List subvolumegroup snapshots", 'perm': 'r' }, { 'cmd': 'fs subvolumegroup snapshot create ' 'name=vol_name,type=CephString ' 'name=group_name,type=CephString ' 'name=snap_name,type=CephString ', 'desc': "Create a snapshot of a CephFS subvolume group in a volume", 'perm': 'rw' }, { 'cmd': 'fs subvolumegroup snapshot rm ' 'name=vol_name,type=CephString ' 'name=group_name,type=CephString ' 'name=snap_name,type=CephString ' 'name=force,type=CephBool,req=false ', 'desc': "Delete a snapshot of a CephFS subvolume group in a volume", 'perm': 'rw' }, { 'cmd': 'fs subvolume snapshot ls ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "List subvolume snapshots", 'perm': 'r' }, { 'cmd': 'fs subvolume snapshot create ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=snap_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "Create a snapshot of a CephFS subvolume in a volume, " "and optionally, in a specific subvolume group", 'perm': 'rw' }, { 'cmd': 'fs subvolume snapshot info ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=snap_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "Get the metadata of a CephFS subvolume snapshot " "and optionally, in a specific subvolume group", 'perm': 'r' }, { 'cmd': 'fs subvolume snapshot rm ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=snap_name,type=CephString ' 'name=group_name,type=CephString,req=false ' 'name=force,type=CephBool,req=false ', 'desc': "Delete a snapshot of a CephFS subvolume in a volume, " "and optionally, in a specific subvolume group", 'perm': 'rw' }, { 'cmd': 'fs subvolume resize ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=new_size,type=CephString,req=true ' 'name=group_name,type=CephString,req=false ' 'name=no_shrink,type=CephBool,req=false ', 'desc': "Resize a CephFS subvolume", 'perm': 'rw' }, { 'cmd': 'fs subvolume pin' ' name=vol_name,type=CephString' ' name=sub_name,type=CephString' ' name=pin_type,type=CephChoices,strings=export|distributed|random' ' name=pin_setting,type=CephString,req=true' ' name=group_name,type=CephString,req=false', 'desc': "Set MDS pinning policy for subvolume", 'perm': 'rw' }, { 'cmd': 'fs subvolume snapshot protect ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=snap_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "(deprecated) Protect snapshot of a CephFS subvolume in a volume, " "and optionally, in a specific subvolume group", 'perm': 'rw' }, { 'cmd': 'fs subvolume snapshot unprotect ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=snap_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "(deprecated) Unprotect a snapshot of a CephFS subvolume in a volume, " "and optionally, in a specific subvolume group", 'perm': 'rw' }, { 'cmd': 'fs subvolume snapshot clone ' 'name=vol_name,type=CephString ' 'name=sub_name,type=CephString ' 'name=snap_name,type=CephString ' 'name=target_sub_name,type=CephString ' 'name=pool_layout,type=CephString,req=false ' 'name=group_name,type=CephString,req=false ' 'name=target_group_name,type=CephString,req=false ', 'desc': "Clone a snapshot to target subvolume", 'perm': 'rw' }, { 'cmd': 'fs clone status ' 'name=vol_name,type=CephString ' 'name=clone_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "Get status on a cloned subvolume.", 'perm': 'r' }, { 'cmd': 'fs clone cancel ' 'name=vol_name,type=CephString ' 'name=clone_name,type=CephString ' 'name=group_name,type=CephString,req=false ', 'desc': "Cancel an pending or ongoing clone operation.", 'perm': 'r' }, { 'cmd': 'nfs export create cephfs ' 'name=fsname,type=CephString ' 'name=clusterid,type=CephString ' 'name=binding,type=CephString ' 'name=readonly,type=CephBool,req=false ' 'name=path,type=CephString,req=false ', 'desc': "Create a cephfs export", 'perm': 'rw' }, { 'cmd': 'nfs export delete ' 'name=clusterid,type=CephString ' 'name=binding,type=CephString ', 'desc': "Delete a cephfs export", 'perm': 'rw' }, { 'cmd': 'nfs export ls ' 'name=clusterid,type=CephString ' 'name=detailed,type=CephBool,req=false ', 'desc': "List exports of a NFS cluster", 'perm': 'r' }, { 'cmd': 'nfs export get ' 'name=clusterid,type=CephString ' 'name=binding,type=CephString ', 'desc': "Fetch a export of a NFS cluster given the pseudo path/binding", 'perm': 'r' }, { 'cmd': 'nfs cluster create ' 'name=type,type=CephString ' f'name=clusterid,type=CephString,goodchars={goodchars} ' 'name=placement,type=CephString,req=false ', 'desc': "Create an NFS Cluster", 'perm': 'rw' }, { 'cmd': 'nfs cluster update ' 'name=clusterid,type=CephString ' 'name=placement,type=CephString ', 'desc': "Updates an NFS Cluster", 'perm': 'rw' }, { 'cmd': 'nfs cluster delete ' 'name=clusterid,type=CephString ', 'desc': "Deletes an NFS Cluster", 'perm': 'rw' }, { 'cmd': 'nfs cluster ls ', 'desc': "List NFS Clusters", 'perm': 'r' }, { 'cmd': 'nfs cluster info ' 'name=clusterid,type=CephString,req=false ', 'desc': "Displays NFS Cluster info", 'perm': 'r' }, { 'cmd': 'nfs cluster config set ' 'name=clusterid,type=CephString ', 'desc': "Set NFS-Ganesha config by `-i <config_file>`", 'perm': 'rw' }, { 'cmd': 'nfs cluster config reset ' 'name=clusterid,type=CephString ', 'desc': "Reset NFS-Ganesha Config to default", 'perm': 'rw' }, # volume ls [recursive] # subvolume ls <volume> # volume authorize/deauthorize # subvolume authorize/deauthorize # volume describe (free space, etc) # volume auth list (vc.get_authorized_ids) # snapshots? # FIXME: we're doing CephFSVolumeClient.recover on every # path where we instantiate and connect a client. Perhaps # keep clients alive longer, or just pass a "don't recover" # flag in if it's the >1st time we connected a particular # volume in the lifetime of this module instance. ] MODULE_OPTIONS = [ Option( 'max_concurrent_clones', type='int', default=4, desc='Number of asynchronous cloner threads', ) ] def __init__(self, *args, **kwargs): self.inited = False # for mypy self.max_concurrent_clones = None self.lock = threading.Lock() super(Module, self).__init__(*args, **kwargs) # Initialize config option members self.config_notify() with self.lock: self.vc = VolumeClient(self) self.fs_export = FSExport(self) self.nfs = NFSCluster(self) self.inited = True def __del__(self): self.vc.shutdown() def shutdown(self): self.vc.shutdown() def config_notify(self): """ This method is called whenever one of our config options is changed. """ with self.lock: for opt in self.MODULE_OPTIONS: setattr( self, opt['name'], # type: ignore self.get_module_option(opt['name'])) # type: ignore self.log.debug(' mgr option %s = %s', opt['name'], getattr(self, opt['name'])) # type: ignore if self.inited: if opt['name'] == "max_concurrent_clones": self.vc.cloner.reconfigure_max_concurrent_clones( self.max_concurrent_clones) def handle_command(self, inbuf, cmd): handler_name = "_cmd_" + cmd['prefix'].replace(" ", "_") try: handler = getattr(self, handler_name) except AttributeError: return -errno.EINVAL, "", "Unknown command" return handler(inbuf, cmd) @mgr_cmd_wrap def _cmd_fs_volume_create(self, inbuf, cmd): vol_id = cmd['name'] placement = cmd.get('placement', '') return self.vc.create_fs_volume(vol_id, placement) @mgr_cmd_wrap def _cmd_fs_volume_rm(self, inbuf, cmd): vol_name = cmd['vol_name'] confirm = cmd.get('yes-i-really-mean-it', None) return self.vc.delete_fs_volume(vol_name, confirm) @mgr_cmd_wrap def _cmd_fs_volume_ls(self, inbuf, cmd): return self.vc.list_fs_volumes() @mgr_cmd_wrap def _cmd_fs_subvolumegroup_create(self, inbuf, cmd): """ :return: a 3-tuple of return code(int), empty string(str), error message (str) """ return self.vc.create_subvolume_group(vol_name=cmd['vol_name'], group_name=cmd['group_name'], pool_layout=cmd.get( 'pool_layout', None), mode=cmd.get('mode', '755'), uid=cmd.get('uid', None), gid=cmd.get('gid', None)) @mgr_cmd_wrap def _cmd_fs_subvolumegroup_rm(self, inbuf, cmd): """ :return: a 3-tuple of return code(int), empty string(str), error message (str) """ return self.vc.remove_subvolume_group(vol_name=cmd['vol_name'], group_name=cmd['group_name'], force=cmd.get('force', False)) @mgr_cmd_wrap def _cmd_fs_subvolumegroup_ls(self, inbuf, cmd): return self.vc.list_subvolume_groups(vol_name=cmd['vol_name']) @mgr_cmd_wrap def _cmd_fs_subvolume_create(self, inbuf, cmd): """ :return: a 3-tuple of return code(int), empty string(str), error message (str) """ return self.vc.create_subvolume( vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], group_name=cmd.get('group_name', None), size=cmd.get('size', None), pool_layout=cmd.get('pool_layout', None), uid=cmd.get('uid', None), gid=cmd.get('gid', None), mode=cmd.get('mode', '755'), namespace_isolated=cmd.get('namespace_isolated', False)) @mgr_cmd_wrap def _cmd_fs_subvolume_rm(self, inbuf, cmd): """ :return: a 3-tuple of return code(int), empty string(str), error message (str) """ return self.vc.remove_subvolume(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], group_name=cmd.get('group_name', None), force=cmd.get('force', False), retain_snapshots=cmd.get( 'retain_snapshots', False)) @mgr_cmd_wrap def _cmd_fs_subvolume_authorize(self, inbuf, cmd): """ :return: a 3-tuple of return code(int), secret key(str), error message (str) """ return self.vc.authorize_subvolume( vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], auth_id=cmd['auth_id'], group_name=cmd.get('group_name', None), access_level=cmd.get('access_level', 'rw'), tenant_id=cmd.get('tenant_id', None), allow_existing_id=cmd.get('allow_existing_id', False)) @mgr_cmd_wrap def _cmd_fs_subvolume_deauthorize(self, inbuf, cmd): """ :return: a 3-tuple of return code(int), empty string(str), error message (str) """ return self.vc.deauthorize_subvolume(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], auth_id=cmd['auth_id'], group_name=cmd.get( 'group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_authorized_list(self, inbuf, cmd): """ :return: a 3-tuple of return code(int), list of authids(json), error message (str) """ return self.vc.authorized_list(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], group_name=cmd.get('group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_evict(self, inbuf, cmd): """ :return: a 3-tuple of return code(int), empyt string(str), error message (str) """ return self.vc.evict(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], auth_id=cmd['auth_id'], group_name=cmd.get('group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_ls(self, inbuf, cmd): return self.vc.list_subvolumes(vol_name=cmd['vol_name'], group_name=cmd.get('group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolumegroup_getpath(self, inbuf, cmd): return self.vc.getpath_subvolume_group(vol_name=cmd['vol_name'], group_name=cmd['group_name']) @mgr_cmd_wrap def _cmd_fs_subvolume_getpath(self, inbuf, cmd): return self.vc.subvolume_getpath(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], group_name=cmd.get( 'group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_info(self, inbuf, cmd): return self.vc.subvolume_info(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], group_name=cmd.get('group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolumegroup_pin(self, inbuf, cmd): return self.vc.pin_subvolume_group(vol_name=cmd['vol_name'], group_name=cmd['group_name'], pin_type=cmd['pin_type'], pin_setting=cmd['pin_setting']) @mgr_cmd_wrap def _cmd_fs_subvolumegroup_snapshot_create(self, inbuf, cmd): return self.vc.create_subvolume_group_snapshot( vol_name=cmd['vol_name'], group_name=cmd['group_name'], snap_name=cmd['snap_name']) @mgr_cmd_wrap def _cmd_fs_subvolumegroup_snapshot_rm(self, inbuf, cmd): return self.vc.remove_subvolume_group_snapshot( vol_name=cmd['vol_name'], group_name=cmd['group_name'], snap_name=cmd['snap_name'], force=cmd.get('force', False)) @mgr_cmd_wrap def _cmd_fs_subvolumegroup_snapshot_ls(self, inbuf, cmd): return self.vc.list_subvolume_group_snapshots( vol_name=cmd['vol_name'], group_name=cmd['group_name']) @mgr_cmd_wrap def _cmd_fs_subvolume_snapshot_create(self, inbuf, cmd): return self.vc.create_subvolume_snapshot(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], snap_name=cmd['snap_name'], group_name=cmd.get( 'group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_snapshot_rm(self, inbuf, cmd): return self.vc.remove_subvolume_snapshot(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], snap_name=cmd['snap_name'], group_name=cmd.get( 'group_name', None), force=cmd.get('force', False)) @mgr_cmd_wrap def _cmd_fs_subvolume_snapshot_info(self, inbuf, cmd): return self.vc.subvolume_snapshot_info(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], snap_name=cmd['snap_name'], group_name=cmd.get( 'group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_snapshot_ls(self, inbuf, cmd): return self.vc.list_subvolume_snapshots(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], group_name=cmd.get( 'group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_resize(self, inbuf, cmd): return self.vc.resize_subvolume(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], new_size=cmd['new_size'], group_name=cmd.get('group_name', None), no_shrink=cmd.get('no_shrink', False)) @mgr_cmd_wrap def _cmd_fs_subvolume_pin(self, inbuf, cmd): return self.vc.subvolume_pin(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], pin_type=cmd['pin_type'], pin_setting=cmd['pin_setting'], group_name=cmd.get('group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_snapshot_protect(self, inbuf, cmd): return self.vc.protect_subvolume_snapshot(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], snap_name=cmd['snap_name'], group_name=cmd.get( 'group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_snapshot_unprotect(self, inbuf, cmd): return self.vc.unprotect_subvolume_snapshot(vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], snap_name=cmd['snap_name'], group_name=cmd.get( 'group_name', None)) @mgr_cmd_wrap def _cmd_fs_subvolume_snapshot_clone(self, inbuf, cmd): return self.vc.clone_subvolume_snapshot( vol_name=cmd['vol_name'], sub_name=cmd['sub_name'], snap_name=cmd['snap_name'], group_name=cmd.get('group_name', None), pool_layout=cmd.get('pool_layout', None), target_sub_name=cmd['target_sub_name'], target_group_name=cmd.get('target_group_name', None)) @mgr_cmd_wrap def _cmd_fs_clone_status(self, inbuf, cmd): return self.vc.clone_status(vol_name=cmd['vol_name'], clone_name=cmd['clone_name'], group_name=cmd.get('group_name', None)) @mgr_cmd_wrap def _cmd_fs_clone_cancel(self, inbuf, cmd): return self.vc.clone_cancel(vol_name=cmd['vol_name'], clone_name=cmd['clone_name'], group_name=cmd.get('group_name', None)) @mgr_cmd_wrap def _cmd_nfs_export_create_cephfs(self, inbuf, cmd): #TODO Extend export creation for rgw. return self.fs_export.create_export(fs_name=cmd['fsname'], cluster_id=cmd['clusterid'], pseudo_path=cmd['binding'], read_only=cmd.get( 'readonly', False), path=cmd.get('path', '/')) @mgr_cmd_wrap def _cmd_nfs_export_delete(self, inbuf, cmd): return self.fs_export.delete_export(cluster_id=cmd['clusterid'], pseudo_path=cmd['binding']) @mgr_cmd_wrap def _cmd_nfs_export_ls(self, inbuf, cmd): return self.fs_export.list_exports(cluster_id=cmd['clusterid'], detailed=cmd.get('detailed', False)) @mgr_cmd_wrap def _cmd_nfs_export_get(self, inbuf, cmd): return self.fs_export.get_export(cluster_id=cmd['clusterid'], pseudo_path=cmd['binding']) @mgr_cmd_wrap def _cmd_nfs_cluster_create(self, inbuf, cmd): return self.nfs.create_nfs_cluster(cluster_id=cmd['clusterid'], export_type=cmd['type'], placement=cmd.get( 'placement', None)) @mgr_cmd_wrap def _cmd_nfs_cluster_update(self, inbuf, cmd): return self.nfs.update_nfs_cluster(cluster_id=cmd['clusterid'], placement=cmd['placement']) @mgr_cmd_wrap def _cmd_nfs_cluster_delete(self, inbuf, cmd): return self.nfs.delete_nfs_cluster(cluster_id=cmd['clusterid']) @mgr_cmd_wrap def _cmd_nfs_cluster_ls(self, inbuf, cmd): return self.nfs.list_nfs_cluster() @mgr_cmd_wrap def _cmd_nfs_cluster_info(self, inbuf, cmd): return self.nfs.show_nfs_cluster_info( cluster_id=cmd.get('clusterid', None)) def _cmd_nfs_cluster_config_set(self, inbuf, cmd): return self.nfs.set_nfs_cluster_config(cluster_id=cmd['clusterid'], nfs_config=inbuf) def _cmd_nfs_cluster_config_reset(self, inbuf, cmd): return self.nfs.reset_nfs_cluster_config(cluster_id=cmd['clusterid'])
class Module(MgrModule): metadata_keys = [ "arch", "ceph_version", "os", "cpu", "kernel_description", "kernel_version", "distro_description", "distro" ] MODULE_OPTIONS = [ Option(name='url', type='str', default='https://telemetry.ceph.com/report'), Option(name='device_url', type='str', default='https://telemetry.ceph.com/device'), Option(name='enabled', type='bool', default=False), Option(name='last_opt_revision', type='int', default=1), Option(name='leaderboard', type='bool', default=False), Option(name='description', type='str', default=None), Option(name='contact', type='str', default=None), Option(name='organization', type='str', default=None), Option(name='proxy', type='str', default=None), Option(name='interval', type='int', default=24, min=8), Option(name='channel_basic', type='bool', default=True, desc='Share basic cluster information (size, version)'), Option(name='channel_ident', type='bool', default=False, desc='Share a user-provided description and/or contact email for the cluster'), Option(name='channel_crash', type='bool', default=True, desc='Share metadata about Ceph daemon crashes (version, stack straces, etc)'), Option(name='channel_device', type='bool', default=True, desc=('Share device health metrics ' '(e.g., SMART data, minus potentially identifying info like serial numbers)')), Option(name='channel_perf', type='bool', default=False, desc='Share perf counter metrics summed across the whole cluster'), ] @property def config_keys(self) -> Dict[str, OptionValue]: return dict((o['name'], o.get('default', None)) for o in self.MODULE_OPTIONS) def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) self.event = Event() self.run = False self.last_upload: Optional[int] = None self.last_report: Dict[str, Any] = dict() self.report_id: Optional[str] = None self.salt: Optional[str] = None self.config_update_module_option() # for mypy which does not run the code if TYPE_CHECKING: self.url = '' self.device_url = '' self.enabled = False self.last_opt_revision = 0 self.leaderboard = '' self.interval = 0 self.proxy = '' self.channel_basic = True self.channel_ident = False self.channel_crash = True self.channel_device = True self.channel_perf = False def config_update_module_option(self) -> None: for opt in self.MODULE_OPTIONS: setattr(self, opt['name'], self.get_module_option(opt['name'])) self.log.debug(' %s = %s', opt['name'], getattr(self, opt['name'])) def config_notify(self) -> None: self.config_update_module_option() # wake up serve() thread self.event.set() def load(self) -> None: last_upload = self.get_store('last_upload', None) if last_upload is None: self.last_upload = None else: self.last_upload = int(last_upload) report_id = self.get_store('report_id', None) if report_id is None: self.report_id = str(uuid.uuid4()) self.set_store('report_id', self.report_id) else: self.report_id = report_id salt = self.get_store('salt', None) if salt is None: self.salt = str(uuid.uuid4()) self.set_store('salt', self.salt) else: self.salt = salt def gather_osd_metadata(self, osd_map: Dict[str, List[Dict[str, int]]]) -> Dict[str, Dict[str, int]]: keys = ["osd_objectstore", "rotational"] keys += self.metadata_keys metadata: Dict[str, Dict[str, int]] = dict() for key in keys: metadata[key] = defaultdict(int) for osd in osd_map['osds']: res = self.get_metadata('osd', str(osd['osd'])) if res is None: self.log.debug('Could not get metadata for osd.%s' % str(osd['osd'])) continue for k, v in res.items(): if k not in keys: continue metadata[k][v] += 1 return metadata def gather_mon_metadata(self, mon_map: Dict[str, List[Dict[str, str]]]) -> Dict[str, Dict[str, int]]: keys = list() keys += self.metadata_keys metadata: Dict[str, Dict[str, int]] = dict() for key in keys: metadata[key] = defaultdict(int) for mon in mon_map['mons']: res = self.get_metadata('mon', mon['name']) if res is None: self.log.debug('Could not get metadata for mon.%s' % (mon['name'])) continue for k, v in res.items(): if k not in keys: continue metadata[k][v] += 1 return metadata def gather_crush_info(self) -> Dict[str, Union[int, bool, List[int], Dict[str, int], Dict[int, int]]]: osdmap = self.get_osdmap() crush_raw = osdmap.get_crush() crush = crush_raw.dump() BucketKeyT = TypeVar('BucketKeyT', int, str) def inc(d: Dict[BucketKeyT, int], k: BucketKeyT) -> None: if k in d: d[k] += 1 else: d[k] = 1 device_classes: Dict[str, int] = {} for dev in crush['devices']: inc(device_classes, dev.get('class', '')) bucket_algs: Dict[str, int] = {} bucket_types: Dict[str, int] = {} bucket_sizes: Dict[int, int] = {} for bucket in crush['buckets']: if '~' in bucket['name']: # ignore shadow buckets continue inc(bucket_algs, bucket['alg']) inc(bucket_types, bucket['type_id']) inc(bucket_sizes, len(bucket['items'])) return { 'num_devices': len(crush['devices']), 'num_types': len(crush['types']), 'num_buckets': len(crush['buckets']), 'num_rules': len(crush['rules']), 'device_classes': list(device_classes.values()), 'tunables': crush['tunables'], 'compat_weight_set': '-1' in crush['choose_args'], 'num_weight_sets': len(crush['choose_args']), 'bucket_algs': bucket_algs, 'bucket_sizes': bucket_sizes, 'bucket_types': bucket_types, } def gather_configs(self) -> Dict[str, List[str]]: # cluster config options cluster = set() r, outb, outs = self.mon_command({ 'prefix': 'config dump', 'format': 'json' }) if r != 0: return {} try: dump = json.loads(outb) except json.decoder.JSONDecodeError: return {} for opt in dump: name = opt.get('name') if name: cluster.add(name) # daemon-reported options (which may include ceph.conf) active = set() ls = self.get("modified_config_options") for opt in ls.get('options', {}): active.add(opt) return { 'cluster_changed': sorted(list(cluster)), 'active_changed': sorted(list(active)), } def get_mempool(self, mode: str = 'separated') -> Dict[str, dict]: # Initialize result dict result: Dict[str, dict] = defaultdict(lambda: defaultdict(int)) # Get list of osd ids from the metadata osd_metadata = self.get('osd_metadata') # Grab output from the "osd.x dump_mempools" command for osd_id in osd_metadata: cmd_dict = { 'prefix': 'dump_mempools', 'id': str(osd_id), 'format': 'json' } r, outb, outs = self.osd_command(cmd_dict) if r != 0: self.log.debug("Invalid command dictionary.") continue else: try: # This is where the mempool will land. dump = json.loads(outb) if mode == 'separated': result["osd." + str(osd_id)] = dump['mempool']['by_pool'] elif mode == 'aggregated': for mem_type in dump['mempool']['by_pool']: result[mem_type]['bytes'] += dump['mempool']['by_pool'][mem_type]['bytes'] result[mem_type]['items'] += dump['mempool']['by_pool'][mem_type]['items'] else: self.log.debug("Incorrect mode specified in get_mempool") except (json.decoder.JSONDecodeError, KeyError) as e: self.log.debug("Error caught: {}".format(e)) return {} return result def get_osd_histograms(self, mode: str = 'separated') -> List[Dict[str, dict]]: # Initialize result dict result: Dict[str, dict] = defaultdict(lambda: defaultdict( lambda: defaultdict( lambda: defaultdict( lambda: defaultdict( lambda: defaultdict(int)))))) # Get list of osd ids from the metadata osd_metadata = self.get('osd_metadata') # Grab output from the "osd.x perf histogram dump" command for osd_id in osd_metadata: cmd_dict = { 'prefix': 'perf histogram dump', 'id': str(osd_id), 'format': 'json' } r, outb, outs = self.osd_command(cmd_dict) # Check for invalid calls if r != 0: self.log.debug("Invalid command dictionary.") continue else: try: # This is where the histograms will land if there are any. dump = json.loads(outb) for histogram in dump['osd']: # Log axis information. There are two axes, each represented # as a dictionary. Both dictionaries are contained inside a # list called 'axes'. axes = [] for axis in dump['osd'][histogram]['axes']: # This is the dict that contains information for an individual # axis. It will be appended to the 'axes' list at the end. axis_dict: Dict[str, Any] = defaultdict() # Collecting information for buckets, min, name, etc. axis_dict['buckets'] = axis['buckets'] axis_dict['min'] = axis['min'] axis_dict['name'] = axis['name'] axis_dict['quant_size'] = axis['quant_size'] axis_dict['scale_type'] = axis['scale_type'] # Collecting ranges; placing them in lists to # improve readability later on. ranges = [] for _range in axis['ranges']: _max, _min = None, None if 'max' in _range: _max = _range['max'] if 'min' in _range: _min = _range['min'] ranges.append([_min, _max]) axis_dict['ranges'] = ranges # Now that 'axis_dict' contains all the appropriate # information for the current axis, append it to the 'axes' list. # There will end up being two axes in the 'axes' list, since the # histograms are 2D. axes.append(axis_dict) # Add the 'axes' list, containing both axes, to result. # At this point, you will see that the name of the key is the string # form of our axes list (str(axes)). This is there so that histograms # with different axis configs will not be combined. # These key names are later dropped when only the values are returned. result[str(axes)][histogram]['axes'] = axes # Collect current values and make sure they are in # integer form. values = [] for value_list in dump['osd'][histogram]['values']: values.append([int(v) for v in value_list]) if mode == 'separated': if 'osds' not in result[str(axes)][histogram]: result[str(axes)][histogram]['osds'] = [] result[str(axes)][histogram]['osds'].append({'osd_id': int(osd_id), 'values': values}) elif mode == 'aggregated': # Aggregate values. If 'values' have already been initialized, # we can safely add. if 'values' in result[str(axes)][histogram]: for i in range (0, len(values)): for j in range (0, len(values[i])): values[i][j] += result[str(axes)][histogram]['values'][i][j] # Add the values to result. result[str(axes)][histogram]['values'] = values # Update num_combined_osds if 'num_combined_osds' not in result[str(axes)][histogram]: result[str(axes)][histogram]['num_combined_osds'] = 1 else: result[str(axes)][histogram]['num_combined_osds'] += 1 else: self.log.error('Incorrect mode specified in get_osd_histograms: {}'.format(mode)) return list() # Sometimes, json errors occur if you give it an empty string. # I am also putting in a catch for a KeyError since it could # happen where the code is assuming that a key exists in the # schema when it doesn't. In either case, we'll handle that # by returning an empty dict. except (json.decoder.JSONDecodeError, KeyError) as e: self.log.debug("Error caught: {}".format(e)) return list() return list(result.values()) def get_io_rate(self) -> dict: return self.get('io_rate') def gather_crashinfo(self) -> List[Dict[str, str]]: crashlist: List[Dict[str, str]] = list() errno, crashids, err = self.remote('crash', 'ls') if errno: return crashlist for crashid in crashids.split(): errno, crashinfo, err = self.remote('crash', 'do_info', crashid) if errno: continue c = json.loads(crashinfo) # redact hostname del c['utsname_hostname'] # entity_name might have more than one '.', beware (etype, eid) = c.get('entity_name', '').split('.', 1) m = hashlib.sha1() assert self.salt m.update(self.salt.encode('utf-8')) m.update(eid.encode('utf-8')) m.update(self.salt.encode('utf-8')) c['entity_name'] = etype + '.' + m.hexdigest() # redact final line of python tracebacks, as the exception # payload may contain identifying information if 'mgr_module' in c and 'backtrace' in c: # backtrace might be empty if len(c['backtrace']) > 0: c['backtrace'][-1] = '<redacted>' crashlist.append(c) return crashlist def gather_perf_counters(self, mode: str = 'separated') -> Dict[str, dict]: # Extract perf counter data with get_all_perf_counters(), a method # from mgr/mgr_module.py. This method returns a nested dictionary that # looks a lot like perf schema, except with some additional fields. # # Example of output, a snapshot of a mon daemon: # "mon.b": { # "bluestore.kv_flush_lat": { # "count": 2431, # "description": "Average kv_thread flush latency", # "nick": "fl_l", # "priority": 8, # "type": 5, # "units": 1, # "value": 88814109 # }, # }, all_perf_counters = self.get_all_perf_counters() # Initialize 'result' dict result: Dict[str, dict] = defaultdict(lambda: defaultdict( lambda: defaultdict(lambda: defaultdict(int)))) for daemon in all_perf_counters: # Calculate num combined daemon types if in aggregated mode if mode == 'aggregated': daemon_type = daemon[0:3] # i.e. 'mds', 'osd', 'rgw' if 'num_combined_daemons' not in result[daemon_type]: result[daemon_type]['num_combined_daemons'] = 1 else: result[daemon_type]['num_combined_daemons'] += 1 for collection in all_perf_counters[daemon]: # Split the collection to avoid redundancy in final report; i.e.: # bluestore.kv_flush_lat, bluestore.kv_final_lat --> # bluestore: kv_flush_lat, kv_final_lat col_0, col_1 = collection.split('.') # Debug log for empty keys. This initially was a problem for prioritycache # perf counters, where the col_0 was empty for certain mon counters: # # "mon.a": { instead of "mon.a": { # "": { "prioritycache": { # "cache_bytes": {...}, "cache_bytes": {...}, # # This log is here to detect any future instances of a similar issue. if (daemon == "") or (col_0 == "") or (col_1 == ""): self.log.debug("Instance of an empty key: {}{}".format(daemon, collection)) if mode == 'separated': # Add value to result result[daemon][col_0][col_1]['value'] = \ all_perf_counters[daemon][collection]['value'] # Check that 'count' exists, as not all counters have a count field. if 'count' in all_perf_counters[daemon][collection]: result[daemon][col_0][col_1]['count'] = \ all_perf_counters[daemon][collection]['count'] elif mode == 'aggregated': # Not every rgw daemon has the same schema. Specifically, each rgw daemon # has a uniquely-named collection that starts off identically (i.e. # "objecter-0x...") then diverges (i.e. "...55f4e778e140.op_rmw"). # This bit of code combines these unique counters all under one rgw instance. # Without this check, the schema would remain separeted out in the final report. if col_0[0:11] == "objecter-0x": col_0 = "objecter-0x" # Check that the value can be incremented. In some cases, # the files are of type 'pair' (real-integer-pair, integer-integer pair). # In those cases, the value is a dictionary, and not a number. # i.e. throttle-msgr_dispatch_throttler-hbserver["wait"] if isinstance(all_perf_counters[daemon][collection]['value'], numbers.Number): result[daemon_type][col_0][col_1]['value'] += \ all_perf_counters[daemon][collection]['value'] # Check that 'count' exists, as not all counters have a count field. if 'count' in all_perf_counters[daemon][collection]: result[daemon_type][col_0][col_1]['count'] += \ all_perf_counters[daemon][collection]['count'] else: self.log.error('Incorrect mode specified in gather_perf_counters: {}'.format(mode)) return {} return result def get_active_channels(self) -> List[str]: r = [] if self.channel_basic: r.append('basic') if self.channel_crash: r.append('crash') if self.channel_device: r.append('device') if self.channel_ident: r.append('ident') if self.channel_perf: r.append('perf') return r def gather_device_report(self) -> Dict[str, Dict[str, Dict[str, str]]]: try: time_format = self.remote('devicehealth', 'get_time_format') except Exception: return {} cutoff = datetime.utcnow() - timedelta(hours=self.interval * 2) min_sample = cutoff.strftime(time_format) devices = self.get('devices')['devices'] # anon-host-id -> anon-devid -> { timestamp -> record } res: Dict[str, Dict[str, Dict[str, str]]] = {} for d in devices: devid = d['devid'] try: # this is a map of stamp -> {device info} m = self.remote('devicehealth', 'get_recent_device_metrics', devid, min_sample) except Exception: continue # anonymize host id try: host = d['location'][0]['host'] except (KeyError, IndexError): continue anon_host = self.get_store('host-id/%s' % host) if not anon_host: anon_host = str(uuid.uuid1()) self.set_store('host-id/%s' % host, anon_host) serial = None for dev, rep in m.items(): rep['host_id'] = anon_host if serial is None and 'serial_number' in rep: serial = rep['serial_number'] # anonymize device id anon_devid = self.get_store('devid-id/%s' % devid) if not anon_devid: # ideally devid is 'vendor_model_serial', # but can also be 'model_serial', 'serial' if '_' in devid: anon_devid = f"{devid.rsplit('_', 1)[0]}_{uuid.uuid1()}" else: anon_devid = str(uuid.uuid1()) self.set_store('devid-id/%s' % devid, anon_devid) self.log.info('devid %s / %s, host %s / %s' % (devid, anon_devid, host, anon_host)) # anonymize the smartctl report itself if serial: m_str = json.dumps(m) m = json.loads(m_str.replace(serial, 'deleted')) if anon_host not in res: res[anon_host] = {} res[anon_host][anon_devid] = m return res def get_latest(self, daemon_type: str, daemon_name: str, stat: str) -> int: data = self.get_counter(daemon_type, daemon_name, stat)[stat] if data: return data[-1][1] else: return 0 def compile_report(self, channels: Optional[List[str]] = None) -> Dict[str, Any]: if not channels: channels = self.get_active_channels() report = { 'leaderboard': self.leaderboard, 'report_version': 1, 'report_timestamp': datetime.utcnow().isoformat(), 'report_id': self.report_id, 'channels': channels, 'channels_available': ALL_CHANNELS, 'license': LICENSE, } if 'ident' in channels: for option in ['description', 'contact', 'organization']: report[option] = getattr(self, option) if 'basic' in channels: mon_map = self.get('mon_map') osd_map = self.get('osd_map') service_map = self.get('service_map') fs_map = self.get('fs_map') df = self.get('df') report['created'] = mon_map['created'] # mons v1_mons = 0 v2_mons = 0 ipv4_mons = 0 ipv6_mons = 0 for mon in mon_map['mons']: for a in mon['public_addrs']['addrvec']: if a['type'] == 'v2': v2_mons += 1 elif a['type'] == 'v1': v1_mons += 1 if a['addr'].startswith('['): ipv6_mons += 1 else: ipv4_mons += 1 report['mon'] = { 'count': len(mon_map['mons']), 'features': mon_map['features'], 'min_mon_release': mon_map['min_mon_release'], 'v1_addr_mons': v1_mons, 'v2_addr_mons': v2_mons, 'ipv4_addr_mons': ipv4_mons, 'ipv6_addr_mons': ipv6_mons, } report['config'] = self.gather_configs() # pools rbd_num_pools = 0 rbd_num_images_by_pool = [] rbd_mirroring_by_pool = [] num_pg = 0 report['pools'] = list() for pool in osd_map['pools']: num_pg += pool['pg_num'] ec_profile = {} if pool['erasure_code_profile']: orig = osd_map['erasure_code_profiles'].get( pool['erasure_code_profile'], {}) ec_profile = { k: orig[k] for k in orig.keys() if k in ['k', 'm', 'plugin', 'technique', 'crush-failure-domain', 'l'] } cast(List[Dict[str, Any]], report['pools']).append( { 'pool': pool['pool'], 'pg_num': pool['pg_num'], 'pgp_num': pool['pg_placement_num'], 'size': pool['size'], 'min_size': pool['min_size'], 'pg_autoscale_mode': pool['pg_autoscale_mode'], 'target_max_bytes': pool['target_max_bytes'], 'target_max_objects': pool['target_max_objects'], 'type': ['', 'replicated', '', 'erasure'][pool['type']], 'erasure_code_profile': ec_profile, 'cache_mode': pool['cache_mode'], } ) if 'rbd' in pool['application_metadata']: rbd_num_pools += 1 ioctx = self.rados.open_ioctx(pool['pool_name']) rbd_num_images_by_pool.append( sum(1 for _ in rbd.RBD().list2(ioctx))) rbd_mirroring_by_pool.append( rbd.RBD().mirror_mode_get(ioctx) != rbd.RBD_MIRROR_MODE_DISABLED) report['rbd'] = { 'num_pools': rbd_num_pools, 'num_images_by_pool': rbd_num_images_by_pool, 'mirroring_by_pool': rbd_mirroring_by_pool} # osds cluster_network = False for osd in osd_map['osds']: if osd['up'] and not cluster_network: front_ip = osd['public_addrs']['addrvec'][0]['addr'].split(':')[0] back_ip = osd['cluster_addrs']['addrvec'][0]['addr'].split(':')[0] if front_ip != back_ip: cluster_network = True report['osd'] = { 'count': len(osd_map['osds']), 'require_osd_release': osd_map['require_osd_release'], 'require_min_compat_client': osd_map['require_min_compat_client'], 'cluster_network': cluster_network, } # crush report['crush'] = self.gather_crush_info() # cephfs report['fs'] = { 'count': len(fs_map['filesystems']), 'feature_flags': fs_map['feature_flags'], 'num_standby_mds': len(fs_map['standbys']), 'filesystems': [], } num_mds = len(fs_map['standbys']) for fsm in fs_map['filesystems']: fs = fsm['mdsmap'] num_sessions = 0 cached_ino = 0 cached_dn = 0 cached_cap = 0 subtrees = 0 rfiles = 0 rbytes = 0 rsnaps = 0 for gid, mds in fs['info'].items(): num_sessions += self.get_latest('mds', mds['name'], 'mds_sessions.session_count') cached_ino += self.get_latest('mds', mds['name'], 'mds_mem.ino') cached_dn += self.get_latest('mds', mds['name'], 'mds_mem.dn') cached_cap += self.get_latest('mds', mds['name'], 'mds_mem.cap') subtrees += self.get_latest('mds', mds['name'], 'mds.subtrees') if mds['rank'] == 0: rfiles = self.get_latest('mds', mds['name'], 'mds.root_rfiles') rbytes = self.get_latest('mds', mds['name'], 'mds.root_rbytes') rsnaps = self.get_latest('mds', mds['name'], 'mds.root_rsnaps') report['fs']['filesystems'].append({ # type: ignore 'max_mds': fs['max_mds'], 'ever_allowed_features': fs['ever_allowed_features'], 'explicitly_allowed_features': fs['explicitly_allowed_features'], 'num_in': len(fs['in']), 'num_up': len(fs['up']), 'num_standby_replay': len( [mds for gid, mds in fs['info'].items() if mds['state'] == 'up:standby-replay']), 'num_mds': len(fs['info']), 'num_sessions': num_sessions, 'cached_inos': cached_ino, 'cached_dns': cached_dn, 'cached_caps': cached_cap, 'cached_subtrees': subtrees, 'balancer_enabled': len(fs['balancer']) > 0, 'num_data_pools': len(fs['data_pools']), 'standby_count_wanted': fs['standby_count_wanted'], 'approx_ctime': fs['created'][0:7], 'files': rfiles, 'bytes': rbytes, 'snaps': rsnaps, }) num_mds += len(fs['info']) report['fs']['total_num_mds'] = num_mds # type: ignore # daemons report['metadata'] = dict(osd=self.gather_osd_metadata(osd_map), mon=self.gather_mon_metadata(mon_map)) # host counts servers = self.list_servers() self.log.debug('servers %s' % servers) hosts = { 'num': len([h for h in servers if h['hostname']]), } for t in ['mon', 'mds', 'osd', 'mgr']: nr_services = sum(1 for host in servers if any(service for service in cast(List[ServiceInfoT], host['services']) if service['type'] == t)) hosts['num_with_' + t] = nr_services report['hosts'] = hosts report['usage'] = { 'pools': len(df['pools']), 'pg_num': num_pg, 'total_used_bytes': df['stats']['total_used_bytes'], 'total_bytes': df['stats']['total_bytes'], 'total_avail_bytes': df['stats']['total_avail_bytes'] } services: DefaultDict[str, int] = defaultdict(int) for key, value in service_map['services'].items(): services[key] += 1 if key == 'rgw': rgw = {} zones = set() zonegroups = set() frontends = set() count = 0 d = value.get('daemons', dict()) for k, v in d.items(): if k == 'summary' and v: rgw[k] = v elif isinstance(v, dict) and 'metadata' in v: count += 1 zones.add(v['metadata']['zone_id']) zonegroups.add(v['metadata']['zonegroup_id']) frontends.add(v['metadata']['frontend_type#0']) # we could actually iterate over all the keys of # the dict and check for how many frontends there # are, but it is unlikely that one would be running # more than 2 supported ones f2 = v['metadata'].get('frontend_type#1', None) if f2: frontends.add(f2) rgw['count'] = count rgw['zones'] = len(zones) rgw['zonegroups'] = len(zonegroups) rgw['frontends'] = list(frontends) # sets aren't json-serializable report['rgw'] = rgw report['services'] = services try: report['balancer'] = self.remote('balancer', 'gather_telemetry') except ImportError: report['balancer'] = { 'active': False } if 'crash' in channels: report['crashes'] = self.gather_crashinfo() if 'perf' in channels: report['perf_counters'] = self.gather_perf_counters('separated') report['stats_per_pool'] = self.get('pg_dump')['pool_stats'] report['stats_per_pg'] = self.get('pg_dump')['pg_stats'] report['io_rate'] = self.get_io_rate() report['osd_perf_histograms'] = self.get_osd_histograms('separated') report['mempool'] = self.get_mempool('separated') # NOTE: We do not include the 'device' channel in this report; it is # sent to a different endpoint. return report def _try_post(self, what: str, url: str, report: Dict[str, Dict[str, str]]) -> Optional[str]: self.log.info('Sending %s to: %s' % (what, url)) proxies = dict() if self.proxy: self.log.info('Send using HTTP(S) proxy: %s', self.proxy) proxies['http'] = self.proxy proxies['https'] = self.proxy try: resp = requests.put(url=url, json=report, proxies=proxies) resp.raise_for_status() except Exception as e: fail_reason = 'Failed to send %s to %s: %s' % (what, url, str(e)) self.log.error(fail_reason) return fail_reason return None class EndPoint(enum.Enum): ceph = 'ceph' device = 'device' def send(self, report: Dict[str, Dict[str, str]], endpoint: Optional[List[EndPoint]] = None) -> Tuple[int, str, str]: if not endpoint: endpoint = [self.EndPoint.ceph, self.EndPoint.device] failed = [] success = [] self.log.debug('Send endpoints %s' % endpoint) for e in endpoint: if e == self.EndPoint.ceph: fail_reason = self._try_post('ceph report', self.url, report) if fail_reason: failed.append(fail_reason) else: now = int(time.time()) self.last_upload = now self.set_store('last_upload', str(now)) success.append('Ceph report sent to {0}'.format(self.url)) self.log.info('Sent report to {0}'.format(self.url)) elif e == self.EndPoint.device: if 'device' in self.get_active_channels(): devices = self.gather_device_report() assert devices num_devs = 0 num_hosts = 0 for host, ls in devices.items(): self.log.debug('host %s devices %s' % (host, ls)) if not len(ls): continue fail_reason = self._try_post('devices', self.device_url, ls) if fail_reason: failed.append(fail_reason) else: num_devs += len(ls) num_hosts += 1 if num_devs: success.append('Reported %d devices across %d hosts' % ( num_devs, len(devices))) if failed: return 1, '', '\n'.join(success + failed) return 0, '', '\n'.join(success) @CLIReadCommand('telemetry status') def status(self) -> Tuple[int, str, str]: ''' Show current configuration ''' r = {} for opt in self.MODULE_OPTIONS: r[opt['name']] = getattr(self, opt['name']) r['last_upload'] = (time.ctime(self.last_upload) if self.last_upload else self.last_upload) return 0, json.dumps(r, indent=4, sort_keys=True), '' @CLICommand('telemetry on') def on(self, license: Optional[str] = None) -> Tuple[int, str, str]: ''' Enable telemetry reports from this cluster ''' if license != LICENSE: return -errno.EPERM, '', f'''Telemetry data is licensed under the {LICENSE_NAME} ({LICENSE_URL}). To enable, add '--license {LICENSE}' to the 'ceph telemetry on' command.''' else: self.set_module_option('enabled', True) self.set_module_option('last_opt_revision', REVISION) return 0, '', '' @CLICommand('telemetry off') def off(self) -> Tuple[int, str, str]: ''' Disable telemetry reports from this cluster ''' self.set_module_option('enabled', False) self.set_module_option('last_opt_revision', 1) return 0, '', '' @CLICommand('telemetry send') def do_send(self, endpoint: Optional[List[EndPoint]] = None, license: Optional[str] = None) -> Tuple[int, str, str]: if self.last_opt_revision < LAST_REVISION_RE_OPT_IN and license != LICENSE: self.log.debug(('A telemetry send attempt while opted-out. ' 'Asking for license agreement')) return -errno.EPERM, '', f'''Telemetry data is licensed under the {LICENSE_NAME} ({LICENSE_URL}). To manually send telemetry data, add '--license {LICENSE}' to the 'ceph telemetry send' command. Please consider enabling the telemetry module with 'ceph telemetry on'.''' else: self.last_report = self.compile_report() return self.send(self.last_report, endpoint) @CLIReadCommand('telemetry show') def show(self, channels: Optional[List[str]] = None) -> Tuple[int, str, str]: ''' Show report of all channels ''' report = self.get_report(channels=channels) # Formatting the perf histograms so they are human-readable. This will change the # ranges and values, which are currently in list form, into strings so that # they are displayed horizontally instead of vertically. try: # Formatting ranges and values in osd_perf_histograms modes_to_be_formatted = ['osd_perf_histograms'] for mode in modes_to_be_formatted: for config in report[mode]: for histogram in config: # Adjust ranges by converting lists into strings for axis in config[histogram]['axes']: for i in range(0, len(axis['ranges'])): axis['ranges'][i] = str(axis['ranges'][i]) # Adjust values by converting lists into strings if mode == 'osd_perf_histograms_aggregated': for i in range(0, len(config[histogram]['values'])): config[histogram]['values'][i] = str(config[histogram]['values'][i]) else: # if mode == 'osd_perf_histograms_separated' for osd in config[histogram]['osds']: for i in range(0, len(osd['values'])): osd['values'][i] = str(osd['values'][i]) except KeyError: # If the perf channel is not enabled, there should be a KeyError since # 'osd_perf_histograms' would not be present in the report. In that case, # the show function should pass as usual without trying to format the # histograms. pass report = json.dumps(report, indent=4, sort_keys=True) if self.channel_device: report += ''' Device report is generated separately. To see it run 'ceph telemetry show-device'.''' return 0, report, '' @CLIReadCommand('telemetry show-device') def show_device(self) -> Tuple[int, str, str]: return 0, json.dumps(self.get_report('device'), indent=4, sort_keys=True), '' @CLIReadCommand('telemetry show-all') def show_all(self) -> Tuple[int, str, str]: return 0, json.dumps(self.get_report('all'), indent=4, sort_keys=True), '' def get_report(self, report_type: str = 'default', channels: Optional[List[str]] = None) -> Dict[str, Any]: if report_type == 'default': return self.compile_report(channels=channels) elif report_type == 'device': return self.gather_device_report() elif report_type == 'all': return {'report': self.compile_report(channels=channels), 'device_report': self.gather_device_report()} return {} def self_test(self) -> None: report = self.compile_report() if len(report) == 0: raise RuntimeError('Report is empty') if 'report_id' not in report: raise RuntimeError('report_id not found in report') def shutdown(self) -> None: self.run = False self.event.set() def refresh_health_checks(self) -> None: health_checks = {} if self.enabled and self.last_opt_revision < LAST_REVISION_RE_OPT_IN: health_checks['TELEMETRY_CHANGED'] = { 'severity': 'warning', 'summary': 'Telemetry requires re-opt-in', 'detail': [ 'telemetry report includes new information; must re-opt-in (or out)' ] } self.set_health_checks(health_checks) def serve(self) -> None: self.load() self.run = True self.log.debug('Waiting for mgr to warm up') time.sleep(10) while self.run: self.event.clear() self.refresh_health_checks() if self.last_opt_revision < LAST_REVISION_RE_OPT_IN: self.log.debug('Not sending report until user re-opts-in') self.event.wait(1800) continue if not self.enabled: self.log.debug('Not sending report until configured to do so') self.event.wait(1800) continue now = int(time.time()) if not self.last_upload or \ (now - self.last_upload) > self.interval * 3600: self.log.info('Compiling and sending report to %s', self.url) try: self.last_report = self.compile_report() except Exception: self.log.exception('Exception while compiling report:') self.send(self.last_report) else: self.log.debug('Interval for sending new report has not expired') sleep = 3600 self.log.debug('Sleeping for %d seconds', sleep) self.event.wait(sleep) @staticmethod def can_run() -> Tuple[bool, str]: return True, ''
class Module(MgrModule, orchestrator.Orchestrator): """An Orchestrator that uses <Ansible Runner Service> to perform operations """ MODULE_OPTIONS = [ # url:port of the Ansible Runner Service Option(name="server_location", type="str", default=""), # Check server identity (True by default) Option(name="verify_server", type="bool", default=True), # Path to an alternative CA bundle Option(name="ca_bundle", type="str", default="") ] def __init__(self, *args, **kwargs): super(Module, self).__init__(*args, **kwargs) self.run = False self.all_completions = [] self._ar_client = None # type: Optional[Client] # TLS certificate and key file names used to connect with the external # Ansible Runner Service self.client_cert_fname = "" self.client_key_fname = "" # used to provide more verbose explanation of errors in status method self.status_message = "" self.all_progress_references = list( ) # type: List[orchestrator.ProgressReference] @property def ar_client(self): # type: () -> Client assert self._ar_client is not None return self._ar_client def available(self): """ Check if Ansible Runner service is working """ available = False msg = "" try: if self._ar_client: available = self.ar_client.is_operative() if not available: msg = "No response from Ansible Runner Service" else: msg = "Not possible to initialize connection with Ansible "\ "Runner service." except AnsibleRunnerServiceError as ex: available = False msg = str(ex) # Add more details to the detected problem if self.status_message: msg = "{}:\n{}".format(msg, self.status_message) return (available, msg) def process(self, completions): """Given a list of Completion instances, progress any which are incomplete. :param completions: list of Completion instances :Returns : True if everything is done. """ if completions: self.log.info("process: completions={0}".format( orchestrator.pretty_print(completions))) def serve(self): """ Mandatory for standby modules """ self.log.info("Starting Ansible Orchestrator module ...") try: # Verify config options and client certificates self.verify_config() # Ansible runner service client self._ar_client = Client( server_url=self.get_module_option('server_location', ''), verify_server=self.get_module_option('verify_server', True), ca_bundle=self.get_module_option('ca_bundle', ''), client_cert=self.client_cert_fname, client_key=self.client_key_fname) except AnsibleRunnerServiceError: self.log.exception( "Ansible Runner Service not available. " "Check external server status/TLS identity or " "connection options. If configuration options changed" " try to disable/enable the module.") self.shutdown() return self.run = True def shutdown(self): self.log.info('Stopping Ansible orchestrator module') self.run = False def get_inventory(self, node_filter=None, refresh=False): """ :param : node_filter instance :param : refresh any cached state :Return : A AnsibleReadOperation instance (Completion Object) """ # Create a new read completion object for execute the playbook op = playbook_operation(client=self.ar_client, playbook=GET_STORAGE_DEVICES_CATALOG_PLAYBOOK, result_pattern="list storage inventory", params={}, output_wizard=ProcessInventory(self.ar_client), event_filter_list=["runner_on_ok"]) self._launch_operation(op) return op def create_osds(self, drive_group): """Create one or more OSDs within a single Drive Group. If no host provided the operation affects all the host in the OSDS role :param drive_group: (ceph.deployment.drive_group.DriveGroupSpec), Drive group with the specification of drives to use """ # Transform drive group specification to Ansible playbook parameters host, osd_spec = dg_2_ansible(drive_group) # Create a new read completion object for execute the playbook op = playbook_operation(client=self.ar_client, playbook=ADD_OSD_PLAYBOOK, result_pattern="", params=osd_spec, querystr_dict={"limit": host}, output_wizard=ProcessPlaybookResult( self.ar_client), event_filter_list=["playbook_on_stats"]) self._launch_operation(op) return op def remove_osds(self, osd_ids, destroy=False): """Remove osd's. :param osd_ids: List of osd's to be removed (List[int]) :param destroy: unsupported. """ assert not destroy extravars = { 'osd_to_kill': ",".join([str(osd_id) for osd_id in osd_ids]), 'ireallymeanit': 'yes' } # Create a new read completion object for execute the playbook op = playbook_operation(client=self.ar_client, playbook=REMOVE_OSD_PLAYBOOK, result_pattern="", params=extravars, output_wizard=ProcessPlaybookResult( self.ar_client), event_filter_list=["playbook_on_stats"]) # Execute the playbook self._launch_operation(op) return op def get_hosts(self): """Provides a list Inventory nodes """ host_ls_op = ars_read(self.ar_client, url=URL_GET_HOSTS, output_wizard=ProcessHostsList(self.ar_client)) return host_ls_op def add_host(self, host): """ Add a host to the Ansible Runner Service inventory in the "orchestrator" group :param host: hostname :returns : orchestrator.Completion """ url_group = URL_MANAGE_GROUP.format(group_name=ORCHESTRATOR_GROUP) try: # Create the orchestrator default group if not exist. # If exists we ignore the error response dummy_response = self.ar_client.http_post(url_group, "", {}) # Here, the default group exists so... # Prepare the operation for adding the new host add_url = URL_ADD_RM_HOSTS.format( host_name=host, inventory_group=ORCHESTRATOR_GROUP) operations = [ars_http_operation(add_url, "post", "", None)] except AnsibleRunnerServiceError as ex: # Problems with the external orchestrator. # Prepare the operation to return the error in a Completion object. self.log.exception("Error checking <orchestrator> group: %s", str(ex)) operations = [ars_http_operation(url_group, "post", "", None)] return ars_change(self.ar_client, operations) def remove_host(self, host): """ Remove a host from all the groups in the Ansible Runner Service inventory. :param host: hostname :returns : orchestrator.Completion """ host_groups = [] # type: List[Any] try: # Get the list of groups where the host is included groups_url = URL_GET_HOST_GROUPS.format(host_name=host) response = self.ar_client.http_get(groups_url) if response.status_code == requests.codes.ok: host_groups = json.loads(response.text)["data"]["groups"] except AnsibleRunnerServiceError: self.log.exception("Error retrieving host groups") raise if not host_groups: # Error retrieving the groups, prepare the completion object to # execute the problematic operation just to provide the error # to the caller operations = [ars_http_operation(groups_url, "get")] else: # Build the operations list operations = list( map( lambda x: ars_http_operation( URL_ADD_RM_HOSTS.format(host_name=host, inventory_group=x), "delete"), host_groups)) return ars_change(self.ar_client, operations) def add_rgw(self, spec): # type: (orchestrator.RGWSpec) -> orchestrator.Completion """ Add a RGW service in the cluster : spec : an Orchestrator.RGWSpec object : returns : Completion object """ # Add the hosts to the inventory in the right group hosts = spec.placement.hosts if not hosts: raise orchestrator.OrchestratorError( "No hosts provided. " "At least one destination host is needed to install the RGW " "service") def set_rgwspec_defaults(spec): spec.rgw_multisite = spec.rgw_multisite if spec.rgw_multisite is not None else True spec.rgw_zonemaster = spec.rgw_zonemaster if spec.rgw_zonemaster is not None else True spec.rgw_zonesecondary = spec.rgw_zonesecondary \ if spec.rgw_zonesecondary is not None else False spec.rgw_multisite_proto = spec.rgw_multisite_proto \ if spec.rgw_multisite_proto is not None else "http" spec.rgw_frontend_port = spec.rgw_frontend_port \ if spec.rgw_frontend_port is not None else 8080 spec.rgw_zonegroup = spec.rgw_zonegroup if spec.rgw_zonegroup is not None else "default" spec.rgw_zone_user = spec.rgw_zone_user if spec.rgw_zone_user is not None else "zone.user" spec.rgw_realm = spec.rgw_realm if spec.rgw_realm is not None else "default" spec.system_access_key = spec.system_access_key \ if spec.system_access_key is not None else spec.genkey(20) spec.system_secret_key = spec.system_secret_key \ if spec.system_secret_key is not None else spec.genkey(40) set_rgwspec_defaults(spec) InventoryGroup("rgws", self.ar_client).update(hosts) # Limit playbook execution to certain hosts limited = ",".join(str(host) for host in hosts) # Add the settings for this service extravars = { k: v for (k, v) in spec.__dict__.items() if k.startswith('rgw_') } extravars['rgw_zone'] = spec.name extravars[ 'rgw_multisite_endpoint_addr'] = spec.rgw_multisite_endpoint_addr extravars[ 'rgw_multisite_endpoints_list'] = spec.rgw_multisite_endpoints_list extravars['rgw_frontend_port'] = str(spec.rgw_frontend_port) # Group hosts by resource (used in rm ops) resource_group = "rgw_zone_{}".format(spec.name) InventoryGroup(resource_group, self.ar_client).update(hosts) # Execute the playbook to create the service op = playbook_operation(client=self.ar_client, playbook=SITE_PLAYBOOK, result_pattern="", params=extravars, querystr_dict={"limit": limited}, output_wizard=ProcessPlaybookResult( self.ar_client), event_filter_list=["playbook_on_stats"]) # Execute the playbook self._launch_operation(op) return op def remove_rgw(self, zone): """ Remove a RGW service providing <zone> :param zone: <zone name> of the RGW ... :returns : Completion object """ # Ansible Inventory group for the kind of service group = "rgws" # get the list of hosts where to remove the service # (hosts in resource group) resource_group = "rgw_zone_{}".format(zone) hosts_list = list(InventoryGroup(resource_group, self.ar_client)) limited = ",".join(hosts_list) # Avoid manual confirmation extravars = {"ireallymeanit": "yes"} # Cleaning of inventory after a sucessful operation clean_inventory = {} clean_inventory[resource_group] = hosts_list clean_inventory[group] = hosts_list # Execute the playbook to remove the service op = playbook_operation(client=self.ar_client, playbook=PURGE_PLAYBOOK, result_pattern="", params=extravars, querystr_dict={"limit": limited}, output_wizard=ProcessPlaybookResult( self.ar_client), event_filter_list=["playbook_on_stats"], clean_hosts_on_success=clean_inventory) # Execute the playbook self._launch_operation(op) return op def _launch_operation(self, ansible_operation): """Launch the operation and add the operation to the completion objects ongoing :ansible_operation: A read/write ansible operation (completion object) """ # Add the operation to the list of things ongoing self.all_completions.append(ansible_operation) def verify_config(self): """Verify mandatory settings for the module and provide help to configure properly the orchestrator """ # Retrieve TLS content to use and check them # First try to get certiticate and key content for this manager instance # ex: mgr/ansible/mgr0/[crt/key] self.log.info("Tying to use configured specific certificate and key" "files for this server") the_crt = self.get_store("{}/{}".format(self.get_mgr_id(), "crt")) the_key = self.get_store("{}/{}".format(self.get_mgr_id(), "key")) if the_crt is None or the_key is None: # If not possible... try to get generic certificates and key content # ex: mgr/ansible/[crt/key] self.log.warning("Specific tls files for this manager not " "configured, trying to use generic files") the_crt = self.get_store("crt") the_key = self.get_store("key") if the_crt is None or the_key is None: self.status_message = "No client certificate configured. Please "\ "set Ansible Runner Service client "\ "certificate and key:\n"\ "ceph ansible set-ssl-certificate-"\ "{key,certificate} -i <file>" self.log.error(self.status_message) return # generate certificate temp files self.client_cert_fname = generate_temp_file("crt", the_crt) self.client_key_fname = generate_temp_file("key", the_key) try: verify_tls_files(self.client_cert_fname, self.client_key_fname) except ServerConfigException as e: self.status_message = str(e) if self.status_message: self.log.error(self.status_message) return # Check module options if not self.get_module_option("server_location", ""): self.status_message = "No Ansible Runner Service base URL "\ "<server_name>:<port>."\ "Try 'ceph config set mgr mgr/{0}/server_location "\ "<server name/ip>:<port>'".format(self.module_name) self.log.error(self.status_message) return if self.get_module_option("verify_server", True): self.status_message = "TLS server identity verification is enabled"\ " by default.Use 'ceph config set mgr mgr/{0}/verify_server False'"\ "to disable it.Use 'ceph config set mgr mgr/{0}/ca_bundle <path>'"\ "to point an alternative CA bundle path used for TLS server "\ "verification".format(self.module_name) self.log.error(self.status_message) return # Everything ok self.status_message = "" #--------------------------------------------------------------------------- # Ansible Orchestrator self-owned commands #--------------------------------------------------------------------------- @CLIWriteCommand("ansible set-ssl-certificate", "name=mgr_id,type=CephString,req=false") def set_tls_certificate(self, mgr_id=None, inbuf=None): """Load tls certificate in mon k-v store """ if inbuf is None: return -errno.EINVAL, \ 'Please specify the certificate file with "-i" option', '' if mgr_id is not None: self.set_store("{}/crt".format(mgr_id), inbuf) else: self.set_store("crt", inbuf) return 0, "SSL certificate updated", "" @CLIWriteCommand("ansible set-ssl-certificate-key", "name=mgr_id,type=CephString,req=false") def set_tls_certificate_key(self, mgr_id=None, inbuf=None): """Load tls certificate key in mon k-v store """ if inbuf is None: return -errno.EINVAL, \ 'Please specify the certificate key file with "-i" option', \ '' if mgr_id is not None: self.set_store("{}/key".format(mgr_id), inbuf) else: self.set_store("key", inbuf) return 0, "SSL certificate key updated", ""
class Module(MgrModule): MODULE_OPTIONS = [ Option( 'allow_m_granularity', type='bool', default=False, desc='allow minute scheduled snapshots', runtime=True, ), Option( 'dump_on_update', type='bool', default=False, desc='dump database to debug log on update', runtime=True, ), ] def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) self._initialized = Event() self.client = SnapSchedClient(self) def resolve_subvolume_path(self, fs: str, subvol: Optional[str], path: str) -> str: if not subvol: return path rc, subvol_path, err = self.remote('fs', 'subvolume', 'getpath', fs, subvol) if rc != 0: # TODO custom exception? raise Exception(f'Could not resolve {path} in {fs}, {subvol}') return subvol_path + path @property def default_fs(self) -> str: fs_map = self.get('fs_map') if fs_map['filesystems']: return fs_map['filesystems'][0]['mdsmap']['fs_name'] else: self.log.error('No filesystem instance could be found.') raise CephfsConnectionException( -errno.ENOENT, "no filesystem found") def has_fs(self, fs_name: str) -> bool: return fs_name in self.client.get_all_filesystems() def serve(self) -> None: self._initialized.set() def handle_command(self, inbuf: str, cmd: Dict[str, str]) -> Tuple[int, str, str]: self._initialized.wait() return -errno.EINVAL, "", "Unknown command" @CLIReadCommand('fs snap-schedule status') def snap_schedule_get(self, path: str = '/', subvol: Optional[str] = None, fs: Optional[str] = None, format: Optional[str] = 'plain') -> Tuple[int, str, str]: ''' List current snapshot schedules ''' use_fs = fs if fs else self.default_fs if not self.has_fs(use_fs): return -errno.EINVAL, '', f"no such filesystem: {use_fs}" try: ret_scheds = self.client.get_snap_schedules(use_fs, path) except CephfsConnectionException as e: return e.to_tuple() if format == 'json': json_report = ','.join([ret_sched.report_json() for ret_sched in ret_scheds]) return 0, f'[{json_report}]', '' return 0, '\n===\n'.join([ret_sched.report() for ret_sched in ret_scheds]), '' @CLIReadCommand('fs snap-schedule list') def snap_schedule_list(self, path: str, subvol: Optional[str] = None, recursive: bool = False, fs: Optional[str] = None, format: Optional[str] = 'plain') -> Tuple[int, str, str]: ''' Get current snapshot schedule for <path> ''' try: use_fs = fs if fs else self.default_fs if not self.has_fs(use_fs): return -errno.EINVAL, '', f"no such filesystem: {use_fs}" scheds = self.client.list_snap_schedules(use_fs, path, recursive) self.log.debug(f'recursive is {recursive}') except CephfsConnectionException as e: return e.to_tuple() if not scheds: if format == 'json': output: Dict[str, str] = {} return 0, json.dumps(output), '' return -errno.ENOENT, '', f'SnapSchedule for {path} not found' if format == 'json': # json_list = ','.join([sched.json_list() for sched in scheds]) schedule_list = [sched.schedule for sched in scheds] retention_list = [sched.retention for sched in scheds] out = {'path': path, 'schedule': schedule_list, 'retention': retention_list} return 0, json.dumps(out), '' return 0, '\n'.join([str(sched) for sched in scheds]), '' @CLIWriteCommand('fs snap-schedule add') def snap_schedule_add(self, path: str, snap_schedule: str, start: Optional[str] = None, fs: Optional[str] = None, subvol: Optional[str] = None) -> Tuple[int, str, str]: ''' Set a snapshot schedule for <path> ''' try: use_fs = fs if fs else self.default_fs if not self.has_fs(use_fs): return -errno.EINVAL, '', f"no such filesystem: {use_fs}" abs_path = self.resolve_subvolume_path(use_fs, subvol, path) self.client.store_snap_schedule(use_fs, abs_path, (abs_path, snap_schedule, use_fs, path, start, subvol)) suc_msg = f'Schedule set for path {path}' except sqlite3.IntegrityError: existing_scheds = self.client.get_snap_schedules(use_fs, path) report = [s.report() for s in existing_scheds] error_msg = f'Found existing schedule {report}' self.log.error(error_msg) return -errno.EEXIST, '', error_msg except ValueError as e: return -errno.ENOENT, '', str(e) except CephfsConnectionException as e: return e.to_tuple() return 0, suc_msg, '' @CLIWriteCommand('fs snap-schedule remove') def snap_schedule_rm(self, path: str, repeat: Optional[str] = None, start: Optional[str] = None, subvol: Optional[str] = None, fs: Optional[str] = None) -> Tuple[int, str, str]: ''' Remove a snapshot schedule for <path> ''' try: use_fs = fs if fs else self.default_fs if not self.has_fs(use_fs): return -errno.EINVAL, '', f"no such filesystem: {use_fs}" abs_path = self.resolve_subvolume_path(use_fs, subvol, path) self.client.rm_snap_schedule(use_fs, abs_path, repeat, start) except CephfsConnectionException as e: return e.to_tuple() except ValueError as e: return -errno.ENOENT, '', str(e) return 0, 'Schedule removed for path {}'.format(path), '' @CLIWriteCommand('fs snap-schedule retention add') def snap_schedule_retention_add(self, path: str, retention_spec_or_period: str, retention_count: Optional[str] = None, fs: Optional[str] = None, subvol: Optional[str] = None) -> Tuple[int, str, str]: ''' Set a retention specification for <path> ''' try: use_fs = fs if fs else self.default_fs if not self.has_fs(use_fs): return -errno.EINVAL, '', f"no such filesystem: {use_fs}" abs_path = self.resolve_subvolume_path(use_fs, subvol, path) self.client.add_retention_spec(use_fs, abs_path, retention_spec_or_period, retention_count) except CephfsConnectionException as e: return e.to_tuple() except ValueError as e: return -errno.ENOENT, '', str(e) return 0, 'Retention added to path {}'.format(path), '' @CLIWriteCommand('fs snap-schedule retention remove') def snap_schedule_retention_rm(self, path: str, retention_spec_or_period: str, retention_count: Optional[str] = None, fs: Optional[str] = None, subvol: Optional[str] = None) -> Tuple[int, str, str]: ''' Remove a retention specification for <path> ''' try: use_fs = fs if fs else self.default_fs if not self.has_fs(use_fs): return -errno.EINVAL, '', f"no such filesystem: {use_fs}" abs_path = self.resolve_subvolume_path(use_fs, subvol, path) self.client.rm_retention_spec(use_fs, abs_path, retention_spec_or_period, retention_count) except CephfsConnectionException as e: return e.to_tuple() except ValueError as e: return -errno.ENOENT, '', str(e) return 0, 'Retention removed from path {}'.format(path), '' @CLIWriteCommand('fs snap-schedule activate') def snap_schedule_activate(self, path: str, repeat: Optional[str] = None, start: Optional[str] = None, subvol: Optional[str] = None, fs: Optional[str] = None) -> Tuple[int, str, str]: ''' Activate a snapshot schedule for <path> ''' try: use_fs = fs if fs else self.default_fs if not self.has_fs(use_fs): return -errno.EINVAL, '', f"no such filesystem: {use_fs}" abs_path = self.resolve_subvolume_path(use_fs, subvol, path) self.client.activate_snap_schedule(use_fs, abs_path, repeat, start) except CephfsConnectionException as e: return e.to_tuple() except ValueError as e: return -errno.ENOENT, '', str(e) return 0, 'Schedule activated for path {}'.format(path), '' @CLIWriteCommand('fs snap-schedule deactivate') def snap_schedule_deactivate(self, path: str, repeat: Optional[str] = None, start: Optional[str] = None, subvol: Optional[str] = None, fs: Optional[str] = None) -> Tuple[int, str, str]: ''' Deactivate a snapshot schedule for <path> ''' try: use_fs = fs if fs else self.default_fs if not self.has_fs(use_fs): return -errno.EINVAL, '', f"no such filesystem: {use_fs}" abs_path = self.resolve_subvolume_path(use_fs, subvol, path) self.client.deactivate_snap_schedule(use_fs, abs_path, repeat, start) except CephfsConnectionException as e: return e.to_tuple() except ValueError as e: return -errno.ENOENT, '', str(e) return 0, 'Schedule deactivated for path {}'.format(path), ''
class Module(MgrModule): MODULE_OPTIONS = [ Option(name=MirrorSnapshotScheduleHandler.MODULE_OPTION_NAME), Option(name=MirrorSnapshotScheduleHandler.MODULE_OPTION_NAME_MAX_CONCURRENT_SNAP_CREATE, type='int', default=10), Option(name=TrashPurgeScheduleHandler.MODULE_OPTION_NAME), ] def __init__(self, *args: Any, **kwargs: Any) -> None: super(Module, self).__init__(*args, **kwargs) self.rados.wait_for_latest_osdmap() self.mirror_snapshot_schedule = MirrorSnapshotScheduleHandler(self) self.perf = PerfHandler(self) self.task = TaskHandler(self) self.trash_purge_schedule = TrashPurgeScheduleHandler(self) @CLIWriteCommand('rbd mirror snapshot schedule add') @with_latest_osdmap def mirror_snapshot_schedule_add(self, level_spec: str, interval: str, start_time: Optional[str] = None) -> Tuple[int, str, str]: """ Add rbd mirror snapshot schedule """ spec = LevelSpec.from_name(self, level_spec, namespace_validator, image_validator) return self.mirror_snapshot_schedule.add_schedule(spec, interval, start_time) @CLIWriteCommand('rbd mirror snapshot schedule remove') @with_latest_osdmap def mirror_snapshot_schedule_remove(self, level_spec: str, interval: Optional[str] = None, start_time: Optional[str] = None) -> Tuple[int, str, str]: """ Remove rbd mirror snapshot schedule """ spec = LevelSpec.from_name(self, level_spec, namespace_validator, image_validator) return self.mirror_snapshot_schedule.remove_schedule(spec, interval, start_time) @CLIReadCommand('rbd mirror snapshot schedule list') @with_latest_osdmap def mirror_snapshot_schedule_list(self, level_spec: str = '') -> Tuple[int, str, str]: """ List rbd mirror snapshot schedule """ spec = LevelSpec.from_name(self, level_spec, namespace_validator, image_validator) return self.mirror_snapshot_schedule.list(spec) @CLIReadCommand('rbd mirror snapshot schedule status') @with_latest_osdmap def mirror_snapshot_schedule_status(self, level_spec: str = '') -> Tuple[int, str, str]: """ Show rbd mirror snapshot schedule status """ spec = LevelSpec.from_name(self, level_spec, namespace_validator, image_validator) return self.mirror_snapshot_schedule.status(spec) @CLIReadCommand('rbd perf image stats') @with_latest_osdmap def perf_image_stats(self, pool_spec: Optional[str] = None, sort_by: Optional[ImageSortBy] = None) -> Tuple[int, str, str]: """ Retrieve current RBD IO performance stats """ with self.perf.lock: sort_by_name = sort_by.name if sort_by else OSD_PERF_QUERY_COUNTERS[0] return self.perf.get_perf_stats(pool_spec, sort_by_name) @CLIReadCommand('rbd perf image counters') @with_latest_osdmap def perf_image_counters(self, pool_spec: Optional[str] = None, sort_by: Optional[ImageSortBy] = None) -> Tuple[int, str, str]: """ Retrieve current RBD IO performance counters """ with self.perf.lock: sort_by_name = sort_by.name if sort_by else OSD_PERF_QUERY_COUNTERS[0] return self.perf.get_perf_counters(pool_spec, sort_by_name) @CLIWriteCommand('rbd task add flatten') @with_latest_osdmap def task_add_flatten(self, image_spec: str) -> Tuple[int, str, str]: """ Flatten a cloned image asynchronously in the background """ with self.task.lock: return self.task.queue_flatten(image_spec) @CLIWriteCommand('rbd task add remove') @with_latest_osdmap def task_add_remove(self, image_spec: str) -> Tuple[int, str, str]: """ Remove an image asynchronously in the background """ with self.task.lock: return self.task.queue_remove(image_spec) @CLIWriteCommand('rbd task add trash remove') @with_latest_osdmap def task_add_trash_remove(self, image_id_spec: str) -> Tuple[int, str, str]: """ Remove an image from the trash asynchronously in the background """ with self.task.lock: return self.task.queue_trash_remove(image_id_spec) @CLIWriteCommand('rbd task add migration execute') @with_latest_osdmap def task_add_migration_execute(self, image_spec: str) -> Tuple[int, str, str]: """ Execute an image migration asynchronously in the background """ with self.task.lock: return self.task.queue_migration_execute(image_spec) @CLIWriteCommand('rbd task add migration commit') @with_latest_osdmap def task_add_migration_commit(self, image_spec: str) -> Tuple[int, str, str]: """ Commit an executed migration asynchronously in the background """ with self.task.lock: return self.task.queue_migration_commit(image_spec) @CLIWriteCommand('rbd task add migration abort') @with_latest_osdmap def task_add_migration_abort(self, image_spec: str) -> Tuple[int, str, str]: """ Abort a prepared migration asynchronously in the background """ with self.task.lock: return self.task.queue_migration_abort(image_spec) @CLIWriteCommand('rbd task cancel') @with_latest_osdmap def task_cancel(self, task_id: str) -> Tuple[int, str, str]: """ Cancel a pending or running asynchronous task """ with self.task.lock: return self.task.task_cancel(task_id) @CLIReadCommand('rbd task list') @with_latest_osdmap def task_list(self, task_id: Optional[str] = None) -> Tuple[int, str, str]: """ List pending or running asynchronous tasks """ with self.task.lock: return self.task.task_list(task_id) @CLIWriteCommand('rbd trash purge schedule add') @with_latest_osdmap def trash_purge_schedule_add(self, level_spec: str, interval: str, start_time: Optional[str] = None) -> Tuple[int, str, str]: """ Add rbd trash purge schedule """ spec = LevelSpec.from_name(self, level_spec, allow_image_level=False) return self.trash_purge_schedule.add_schedule(spec, interval, start_time) @CLIWriteCommand('rbd trash purge schedule remove') @with_latest_osdmap def trash_purge_schedule_remove(self, level_spec: str, interval: Optional[str] = None, start_time: Optional[str] = None) -> Tuple[int, str, str]: """ Remove rbd trash purge schedule """ spec = LevelSpec.from_name(self, level_spec, allow_image_level=False) return self.trash_purge_schedule.remove_schedule(spec, interval, start_time) @CLIReadCommand('rbd trash purge schedule list') @with_latest_osdmap def trash_purge_schedule_list(self, level_spec: str = '') -> Tuple[int, str, str]: """ List rbd trash purge schedule """ spec = LevelSpec.from_name(self, level_spec, allow_image_level=False) return self.trash_purge_schedule.list(spec) @CLIReadCommand('rbd trash purge schedule status') @with_latest_osdmap def trash_purge_schedule_status(self, level_spec: str = '') -> Tuple[int, str, str]: """ Show rbd trash purge schedule status """ spec = LevelSpec.from_name(self, level_spec, allow_image_level=False) return self.trash_purge_schedule.status(spec)