def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]: assert self.TYPE == daemon_spec.daemon_type deps = [] # type: List[str] prom_services = [] # type: List[str] for dd in self.mgr.cache.get_daemons_by_service('prometheus'): assert dd.hostname is not None addr = dd.ip if dd.ip else self._inventory_get_fqdn(dd.hostname) port = dd.ports[0] if dd.ports else 9095 prom_services.append(build_url(scheme='http', host=addr, port=port)) deps.append(dd.name()) daemons = self.mgr.cache.get_daemons_by_service('mgr') loki_host = '' assert daemons is not None if daemons != []: assert daemons[0].hostname is not None addr = daemons[0].ip if daemons[0].ip else self._inventory_get_fqdn(daemons[0].hostname) loki_host = build_url(scheme='http', host=addr, port=3100) grafana_data_sources = self.mgr.template.render( 'services/grafana/ceph-dashboard.yml.j2', {'hosts': prom_services, 'loki_host': loki_host}) cert = self.mgr.get_store('grafana_crt') pkey = self.mgr.get_store('grafana_key') if cert and pkey: try: verify_tls(cert, pkey) except ServerConfigException as e: logger.warning('Provided grafana TLS certificates invalid: %s', str(e)) cert, pkey = None, None if not (cert and pkey): cert, pkey = create_self_signed_cert('Ceph', 'cephadm') self.mgr.set_store('grafana_crt', cert) self.mgr.set_store('grafana_key', pkey) if 'dashboard' in self.mgr.get('mgr_map')['modules']: self.mgr.check_mon_command({ 'prefix': 'dashboard set-grafana-api-ssl-verify', 'value': 'false', }) spec: GrafanaSpec = cast( GrafanaSpec, self.mgr.spec_store.active_specs[daemon_spec.service_name]) grafana_ini = self.mgr.template.render( 'services/grafana/grafana.ini.j2', { 'initial_admin_password': spec.initial_admin_password, 'http_port': daemon_spec.ports[0] if daemon_spec.ports else self.DEFAULT_SERVICE_PORT, 'http_addr': daemon_spec.ip if daemon_spec.ip else '' }) config_file = { 'files': { "grafana.ini": grafana_ini, 'provisioning/datasources/ceph-dashboard.yml': grafana_data_sources, 'certs/cert_file': '# generated by cephadm\n%s' % cert, 'certs/cert_key': '# generated by cephadm\n%s' % pkey, } } return config_file, sorted(deps)
def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]: assert self.TYPE == daemon_spec.daemon_type deps: List[str] = [] default_webhook_urls: List[str] = [] spec = cast(AlertManagerSpec, self.mgr.spec_store[daemon_spec.service_name].spec) user_data = spec.user_data if 'default_webhook_urls' in user_data and isinstance( user_data['default_webhook_urls'], list): default_webhook_urls.extend(user_data['default_webhook_urls']) # dashboard(s) dashboard_urls: List[str] = [] mgr_map = self.mgr.get('mgr_map') port = None proto = None # http: or https: url = mgr_map.get('services', {}).get('dashboard', None) if url: dashboard_urls.append(url) p_result = urlparse(url) proto = p_result.scheme port = p_result.port # scan all mgrs to generate deps and to get standbys too. # assume that they are all on the same port as the active mgr. for dd in self.mgr.cache.get_daemons_by_service('mgr'): # we consider mgr a dep even if the dashboard is disabled # in order to be consistent with _calc_daemon_deps(). deps.append(dd.name()) if not port: continue if dd.daemon_id == self.mgr.get_mgr_id(): continue assert dd.hostname is not None addr = self.mgr.inventory.get_addr(dd.hostname) dashboard_urls.append(build_url(scheme=proto, host=addr, port=port)) context = { 'dashboard_urls': dashboard_urls, 'default_webhook_urls': default_webhook_urls } yml = self.mgr.template.render('services/alertmanager/alertmanager.yml.j2', context) peers = [] port = 9094 for dd in self.mgr.cache.get_daemons_by_service('alertmanager'): assert dd.hostname is not None deps.append(dd.name()) addr = self.mgr.inventory.get_addr(dd.hostname) peers.append(build_url(host=addr, port=port).lstrip('/')) return { "files": { "alertmanager.yml": yml }, "peers": peers }, sorted(deps)
def request_begin(self): req = cherrypy.request user = JwtManager.get_username() # Log the request. self.logger.debug('[%s:%s] [%s] [%s] %s', req.remote.ip, req.remote.port, req.method, user, req.path_info) # Audit the request. if Settings.AUDIT_API_ENABLED and req.method not in ['GET']: url = build_url(req.remote.ip, scheme=req.scheme, port=req.remote.port) msg = '[DASHBOARD] from=\'{}\' path=\'{}\' method=\'{}\' ' \ 'user=\'{}\''.format(url, req.path_info, req.method, user) if Settings.AUDIT_API_LOG_PAYLOAD: params = dict(req.params or {}, **get_request_body_params(req)) # Hide sensitive data like passwords, secret keys, ... # Extend the list of patterns to search for if necessary. # Currently parameters like this are processed: # - secret_key # - user_password # - new_passwd_to_login keys = [] for key in ['password', 'passwd', 'secret']: keys.extend([x for x in params.keys() if key in x]) for key in keys: params[key] = '***' msg = '{} params=\'{}\''.format(msg, json.dumps(params)) mgr.cluster_log('audit', mgr.ClusterLogPrio.INFO, msg)
def alertmgr_sd_config(self) -> List[Dict[str, Collection[str]]]: """Return <http_sd_config> compatible prometheus config for mgr alertmanager service.""" srv_entries = [] for dd in self.mgr.cache.get_daemons_by_service('alertmanager'): assert dd.hostname is not None addr = dd.ip if dd.ip else self.mgr.inventory.get_addr(dd.hostname) port = dd.ports[0] if dd.ports else 9093 srv_entries.append('{}'.format(build_url(host=addr, port=port).lstrip('/'))) return [{"targets": srv_entries, "labels": {}}]
def config_dashboard(self, daemon_descrs: List[DaemonDescription]) -> None: dd = self.get_active_daemon(daemon_descrs) assert dd.hostname is not None addr = dd.ip if dd.ip else self._inventory_get_fqdn(dd.hostname) port = dd.ports[0] if dd.ports else self.DEFAULT_SERVICE_PORT service_url = build_url(scheme='http', host=addr, port=port) self._set_service_url_on_dashboard( 'Prometheus', 'dashboard get-prometheus-api-host', 'dashboard set-prometheus-api-host', service_url)
def config_dashboard(self, daemon_descrs: List[DaemonDescription]) -> None: # TODO: signed cert dd = self.get_active_daemon(daemon_descrs) assert dd.hostname is not None addr = dd.ip if dd.ip else self._inventory_get_addr(dd.hostname) port = dd.ports[0] if dd.ports else self.DEFAULT_SERVICE_PORT service_url = build_url(scheme='https', host=addr, port=port) self._set_service_url_on_dashboard('Grafana', 'dashboard get-grafana-api-url', 'dashboard set-grafana-api-url', service_url)
def node_exporter_sd_config(self) -> List[Dict[str, Collection[str]]]: """Return <http_sd_config> compatible prometheus config for node-exporter service.""" srv_entries = [] for dd in self.mgr.cache.get_daemons_by_service('node-exporter'): assert dd.hostname is not None addr = dd.ip if dd.ip else self.mgr.inventory.get_addr(dd.hostname) port = dd.ports[0] if dd.ports else 9100 srv_entries.append({ 'targets': [build_url(host=addr, port=port).lstrip('/')], 'labels': {'instance': dd.hostname} }) return srv_entries
def prepare_create( self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec: assert self.TYPE == daemon_spec.daemon_type collectors = [] for dd in self.mgr.cache.get_daemons_by_type( JaegerCollectorService.TYPE): # scrape jaeger-collector nodes assert dd.hostname is not None port = dd.ports[ 0] if dd.ports else JaegerCollectorService.DEFAULT_SERVICE_PORT url = build_url(host=dd.hostname, port=port).lstrip('/') collectors.append(url) daemon_spec.final_config = {'collector_nodes': ",".join(collectors)} return daemon_spec
def __init__(self, host: str, port: int, client_name: Optional[str] = None, ssl: bool = False, auth: Optional[AuthBase] = None, ssl_verify: bool = True) -> None: super(RestClient, self).__init__() self.client_name = client_name if client_name else '' self.host = host self.port = port self.base_url = build_url(scheme='https' if ssl else 'http', host=host, port=port) logger.debug("REST service base URL: %s", self.base_url) self.headers = {'Accept': 'application/json'} self.auth = auth self.session = TimeoutRequestsSession() self.session.verify = ssl_verify
def __init__(self, access_key: str, secret_key: str, daemon_name: str, user_id: Optional[str] = None) -> None: try: daemon = RgwClient._daemons[daemon_name] except KeyError as error: raise DashboardException( msg='RGW Daemon not found: {}'.format(error), http_status_code=404, component='rgw') ssl_verify = Settings.RGW_API_SSL_VERIFY self.admin_path = Settings.RGW_API_ADMIN_RESOURCE self.service_url = build_url(host=daemon.host, port=daemon.port) self.auth = S3Auth(access_key, secret_key, service_url=self.service_url) super(RgwClient, self).__init__(daemon.host, daemon.port, 'RGW', daemon.ssl, self.auth, ssl_verify=ssl_verify) self.got_keys_from_config = not user_id try: self.userid = self._get_user_id(self.admin_path) if self.got_keys_from_config \ else user_id except RequestException as error: logger.exception(error) msg = 'Error connecting to Object Gateway' if error.status_code == 404: msg = '{}: {}'.format(msg, str(error)) raise DashboardException(msg=msg, http_status_code=error.status_code, component='rgw') self.daemon = daemon logger.info( "Created new connection: daemon=%s, host=%s, port=%s, ssl=%d, sslverify=%d", daemon.name, daemon.host, daemon.port, daemon.ssl, ssl_verify)
def get_elasticsearch_nodes(service: CephadmService, daemon_spec: CephadmDaemonDeploySpec) -> List[str]: elasticsearch_nodes = [] for dd in service.mgr.cache.get_daemons_by_type(ElasticSearchService.TYPE): assert dd.hostname is not None addr = dd.ip if dd.ip else service.mgr.inventory.get_addr(dd.hostname) port = dd.ports[ 0] if dd.ports else ElasticSearchService.DEFAULT_SERVICE_PORT url = build_url(host=addr, port=port).lstrip('/') elasticsearch_nodes.append(f'http://{url}') if len(elasticsearch_nodes) == 0: # takes elasticsearch address from TracingSpec spec: TracingSpec = cast( TracingSpec, service.mgr.spec_store.active_specs[daemon_spec.service_name]) assert spec.es_nodes is not None urls = spec.es_nodes.split(",") for url in urls: elasticsearch_nodes.append(f'http://{url}') return elasticsearch_nodes
def _serve(self): # Load stored authentication keys self.refresh_keys() jsonify._instance = jsonify.GenericJSON( sort_keys=True, indent=4, separators=(',', ': '), ) server_addr = self.get_localized_module_option('server_addr', '::') if server_addr is None: raise CannotServe( 'no server_addr configured; try "ceph config-key set mgr/restful/server_addr <ip>"' ) server_port = int( self.get_localized_module_option('server_port', '8003')) self.log.info('server_addr: %s server_port: %d', server_addr, server_port) cert = self.get_localized_store("crt") if cert is not None: cert_tmp = tempfile.NamedTemporaryFile() cert_tmp.write(cert.encode('utf-8')) cert_tmp.flush() cert_fname = cert_tmp.name else: cert_fname = self.get_localized_store('crt_file') pkey = self.get_localized_store("key") if pkey is not None: pkey_tmp = tempfile.NamedTemporaryFile() pkey_tmp.write(pkey.encode('utf-8')) pkey_tmp.flush() pkey_fname = pkey_tmp.name else: pkey_fname = self.get_localized_module_option('key_file') self.enable_auth = self.get_localized_module_option( 'enable_auth', True) if not cert_fname or not pkey_fname: raise CannotServe('no certificate configured') if not os.path.isfile(cert_fname): raise CannotServe('certificate %s does not exist' % cert_fname) if not os.path.isfile(pkey_fname): raise CannotServe('private key %s does not exist' % pkey_fname) # Publish the URI that others may use to access the service we're # about to start serving addr = self.get_mgr_ip() if server_addr == "::" else server_addr self.set_uri(build_url(scheme='https', host=addr, port=server_port)) # Create the HTTPS werkzeug server serving pecan app self.server = make_server( host=server_addr, port=server_port, app=make_app( root='restful.api.Root', hooks=[ErrorHook()], # use a callable if pecan >= 0.3.2 ), ssl_context=(cert_fname, pkey_fname), ) sock_fd_flag = fcntl.fcntl(self.server.socket.fileno(), fcntl.F_GETFD) if not (sock_fd_flag & fcntl.FD_CLOEXEC): self.log.debug("set server socket close-on-exec") fcntl.fcntl(self.server.socket.fileno(), fcntl.F_SETFD, sock_fd_flag | fcntl.FD_CLOEXEC) if self.stop_server: self.log.debug('made server, but stop flag set') else: self.log.debug('made server, serving forever') self.server.serve_forever()
def _configure(self): """ Configure CherryPy and initialize self.url_prefix :returns our URI """ server_addr = self.get_localized_module_option( # type: ignore 'server_addr', get_default_addr()) use_ssl = self.get_localized_module_option('ssl', True) # type: ignore if not use_ssl: server_port = self.get_localized_module_option( 'server_port', 8080) # type: ignore else: server_port = self.get_localized_module_option( 'ssl_server_port', 8443) # type: ignore if server_addr is None: raise ServerConfigException( 'no server_addr configured; ' 'try "ceph config set mgr mgr/{}/{}/server_addr <ip>"'.format( self.module_name, self.get_mgr_id())) # type: ignore self.log.info( 'server: ssl=%s host=%s port=%d', 'yes' if use_ssl else 'no', # type: ignore server_addr, server_port) # Initialize custom handlers. cherrypy.tools.authenticate = AuthManagerTool() cherrypy.tools.plugin_hooks_filter_request = cherrypy.Tool( 'before_handler', lambda: PLUGIN_MANAGER.hook.filter_request_before_handler( request=cherrypy.request), priority=1) cherrypy.tools.request_logging = RequestLoggingTool() cherrypy.tools.dashboard_exception_handler = HandlerWrapperTool( dashboard_exception_handler, priority=31) cherrypy.log.access_log.propagate = False cherrypy.log.error_log.propagate = False # Apply the 'global' CherryPy configuration. config = { 'engine.autoreload.on': False, 'server.socket_host': server_addr, 'server.socket_port': int(server_port), 'error_page.default': json_error_page, 'tools.request_logging.on': True, 'tools.gzip.on': True, 'tools.gzip.mime_types': [ # text/html and text/plain are the default types to compress 'text/html', 'text/plain', # We also want JSON and JavaScript to be compressed 'application/json', 'application/*+json', 'application/javascript', ], 'tools.json_in.on': True, 'tools.json_in.force': True, 'tools.plugin_hooks_filter_request.on': True, } if use_ssl: # SSL initialization cert = self.get_localized_store("crt") # type: ignore if cert is not None: self.cert_tmp = tempfile.NamedTemporaryFile() self.cert_tmp.write(cert.encode('utf-8')) self.cert_tmp.flush() # cert_tmp must not be gc'ed cert_fname = self.cert_tmp.name else: cert_fname = self.get_localized_module_option( 'crt_file') # type: ignore pkey = self.get_localized_store("key") # type: ignore if pkey is not None: self.pkey_tmp = tempfile.NamedTemporaryFile() self.pkey_tmp.write(pkey.encode('utf-8')) self.pkey_tmp.flush() # pkey_tmp must not be gc'ed pkey_fname = self.pkey_tmp.name else: pkey_fname = self.get_localized_module_option( 'key_file') # type: ignore verify_tls_files(cert_fname, pkey_fname) # Create custom SSL context to disable TLS 1.0 and 1.1. context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) context.load_cert_chain(cert_fname, pkey_fname) if sys.version_info >= (3, 7): context.minimum_version = ssl.TLSVersion.TLSv1_2 else: context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 config['server.ssl_module'] = 'builtin' config['server.ssl_certificate'] = cert_fname config['server.ssl_private_key'] = pkey_fname config['server.ssl_context'] = context self.update_cherrypy_config(config) self._url_prefix = prepare_url_prefix( self.get_module_option( # type: ignore 'url_prefix', default='')) if server_addr in ['::', '0.0.0.0']: server_addr = self.get_mgr_ip() # type: ignore base_url = build_url( scheme='https' if use_ssl else 'http', host=server_addr, port=server_port, ) uri = f'{base_url}{self.url_prefix}/' return uri
def generate_config( self, daemon_spec: CephadmDaemonDeploySpec, ) -> Tuple[Dict[str, Any], List[str]]: assert self.TYPE == daemon_spec.daemon_type deps = [] # type: List[str] # scrape mgrs mgr_scrape_list = [] mgr_map = self.mgr.get('mgr_map') port = cast(int, self.mgr.get_module_option_ex( 'prometheus', 'server_port', self.DEFAULT_MGR_PROMETHEUS_PORT)) deps.append(str(port)) t = mgr_map.get('services', {}).get('prometheus', None) if t: p_result = urlparse(t) # urlparse .hostname removes '[]' from the hostname in case # of ipv6 addresses so if this is the case then we just # append the brackets when building the final scrape endpoint if '[' in p_result.netloc and ']' in p_result.netloc: mgr_scrape_list.append(f"[{p_result.hostname}]:{port}") else: mgr_scrape_list.append(f"{p_result.hostname}:{port}") # scan all mgrs to generate deps and to get standbys too. # assume that they are all on the same port as the active mgr. for dd in self.mgr.cache.get_daemons_by_service('mgr'): # we consider the mgr a dep even if the prometheus module is # disabled in order to be consistent with _calc_daemon_deps(). deps.append(dd.name()) if not port: continue if dd.daemon_id == self.mgr.get_mgr_id(): continue assert dd.hostname is not None addr = self._inventory_get_fqdn(dd.hostname) mgr_scrape_list.append(build_url(host=addr, port=port).lstrip('/')) # scrape node exporters nodes = [] for dd in self.mgr.cache.get_daemons_by_service('node-exporter'): assert dd.hostname is not None deps.append(dd.name()) addr = dd.ip if dd.ip else self._inventory_get_fqdn(dd.hostname) port = dd.ports[0] if dd.ports else 9100 nodes.append({ 'hostname': dd.hostname, 'url': build_url(host=addr, port=port).lstrip('/') }) # scrape alert managers alertmgr_targets = [] for dd in self.mgr.cache.get_daemons_by_service('alertmanager'): assert dd.hostname is not None deps.append(dd.name()) addr = dd.ip if dd.ip else self._inventory_get_fqdn(dd.hostname) port = dd.ports[0] if dd.ports else 9093 alertmgr_targets.append("'{}'".format(build_url(host=addr, port=port).lstrip('/'))) # scrape haproxies haproxy_targets = [] for dd in self.mgr.cache.get_daemons_by_type('ingress'): if dd.service_name() in self.mgr.spec_store: spec = cast(IngressSpec, self.mgr.spec_store[dd.service_name()].spec) assert dd.hostname is not None deps.append(dd.name()) if dd.daemon_type == 'haproxy': addr = self._inventory_get_fqdn(dd.hostname) haproxy_targets.append({ "url": f"'{build_url(host=addr, port=spec.monitor_port).lstrip('/')}'", "service": dd.service_name(), }) # generate the prometheus configuration context = { 'alertmgr_targets': alertmgr_targets, 'mgr_scrape_list': mgr_scrape_list, 'haproxy_targets': haproxy_targets, 'nodes': nodes, } r = { 'files': { 'prometheus.yml': self.mgr.template.render( 'services/prometheus/prometheus.yml.j2', context) } } # include alerts, if present in the container if os.path.exists(self.mgr.prometheus_alerts_path): with open(self.mgr.prometheus_alerts_path, 'r', encoding='utf-8') as f: alerts = f.read() r['files']['/etc/prometheus/alerting/ceph_alerts.yml'] = alerts return r, sorted(deps)
def generate_config( self, daemon_spec: CephadmDaemonDeploySpec ) -> Tuple[Dict[str, Any], List[str]]: assert self.TYPE == daemon_spec.daemon_type deps: List[str] = [] default_webhook_urls: List[str] = [] spec = cast(AlertManagerSpec, self.mgr.spec_store[daemon_spec.service_name].spec) try: secure = spec.secure except AttributeError: secure = False user_data = spec.user_data if 'default_webhook_urls' in user_data and isinstance( user_data['default_webhook_urls'], list): default_webhook_urls.extend(user_data['default_webhook_urls']) # dashboard(s) dashboard_urls: List[str] = [] snmp_gateway_urls: List[str] = [] mgr_map = self.mgr.get('mgr_map') port = None proto = None # http: or https: url = mgr_map.get('services', {}).get('dashboard', None) if url: p_result = urlparse(url.rstrip('/')) hostname = socket.getfqdn(p_result.hostname) try: ip = ipaddress.ip_address(hostname) except ValueError: pass else: if ip.version == 6: hostname = f'[{hostname}]' dashboard_urls.append( f'{p_result.scheme}://{hostname}:{p_result.port}{p_result.path}' ) proto = p_result.scheme port = p_result.port # scan all mgrs to generate deps and to get standbys too. # assume that they are all on the same port as the active mgr. for dd in self.mgr.cache.get_daemons_by_service('mgr'): # we consider mgr a dep even if the dashboard is disabled # in order to be consistent with _calc_daemon_deps(). deps.append(dd.name()) if not port: continue if dd.daemon_id == self.mgr.get_mgr_id(): continue assert dd.hostname is not None addr = self._inventory_get_fqdn(dd.hostname) dashboard_urls.append( build_url(scheme=proto, host=addr, port=port).rstrip('/')) for dd in self.mgr.cache.get_daemons_by_service('snmp-gateway'): assert dd.hostname is not None assert dd.ports addr = dd.ip if dd.ip else self._inventory_get_fqdn(dd.hostname) deps.append(dd.name()) snmp_gateway_urls.append( build_url(scheme='http', host=addr, port=dd.ports[0], path='/alerts')) context = { 'dashboard_urls': dashboard_urls, 'default_webhook_urls': default_webhook_urls, 'snmp_gateway_urls': snmp_gateway_urls, 'secure': secure, } yml = self.mgr.template.render( 'services/alertmanager/alertmanager.yml.j2', context) peers = [] port = 9094 for dd in self.mgr.cache.get_daemons_by_service('alertmanager'): assert dd.hostname is not None deps.append(dd.name()) addr = self._inventory_get_fqdn(dd.hostname) peers.append(build_url(host=addr, port=port).lstrip('/')) return { "files": { "alertmanager.yml": yml }, "peers": peers }, sorted(deps)