def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]:
assert self.TYPE == daemon_spec.daemon_type
deps = [] # type: List[str]
prom_services = [] # type: List[str]
for dd in self.mgr.cache.get_daemons_by_service('prometheus'):
assert dd.hostname is not None
addr = dd.ip if dd.ip else self._inventory_get_fqdn(dd.hostname)
port = dd.ports[0] if dd.ports else 9095
prom_services.append(build_url(scheme='http', host=addr, port=port))
deps.append(dd.name())
daemons = self.mgr.cache.get_daemons_by_service('mgr')
loki_host = ''
assert daemons is not None
if daemons != []:
assert daemons[0].hostname is not None
addr = daemons[0].ip if daemons[0].ip else self._inventory_get_fqdn(daemons[0].hostname)
loki_host = build_url(scheme='http', host=addr, port=3100)
grafana_data_sources = self.mgr.template.render(
'services/grafana/ceph-dashboard.yml.j2', {'hosts': prom_services, 'loki_host': loki_host})
cert = self.mgr.get_store('grafana_crt')
pkey = self.mgr.get_store('grafana_key')
if cert and pkey:
try:
verify_tls(cert, pkey)
except ServerConfigException as e:
logger.warning('Provided grafana TLS certificates invalid: %s', str(e))
cert, pkey = None, None
if not (cert and pkey):
cert, pkey = create_self_signed_cert('Ceph', 'cephadm')
self.mgr.set_store('grafana_crt', cert)
self.mgr.set_store('grafana_key', pkey)
if 'dashboard' in self.mgr.get('mgr_map')['modules']:
self.mgr.check_mon_command({
'prefix': 'dashboard set-grafana-api-ssl-verify',
'value': 'false',
})
spec: GrafanaSpec = cast(
GrafanaSpec, self.mgr.spec_store.active_specs[daemon_spec.service_name])
grafana_ini = self.mgr.template.render(
'services/grafana/grafana.ini.j2', {
'initial_admin_password': spec.initial_admin_password,
'http_port': daemon_spec.ports[0] if daemon_spec.ports else self.DEFAULT_SERVICE_PORT,
'http_addr': daemon_spec.ip if daemon_spec.ip else ''
})
config_file = {
'files': {
"grafana.ini": grafana_ini,
'provisioning/datasources/ceph-dashboard.yml': grafana_data_sources,
'certs/cert_file': '# generated by cephadm\n%s' % cert,
'certs/cert_key': '# generated by cephadm\n%s' % pkey,
}
}
return config_file, sorted(deps)
def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]:
assert self.TYPE == daemon_spec.daemon_type
deps: List[str] = []
default_webhook_urls: List[str] = []
spec = cast(AlertManagerSpec, self.mgr.spec_store[daemon_spec.service_name].spec)
user_data = spec.user_data
if 'default_webhook_urls' in user_data and isinstance(
user_data['default_webhook_urls'], list):
default_webhook_urls.extend(user_data['default_webhook_urls'])
# dashboard(s)
dashboard_urls: List[str] = []
mgr_map = self.mgr.get('mgr_map')
port = None
proto = None # http: or https:
url = mgr_map.get('services', {}).get('dashboard', None)
if url:
dashboard_urls.append(url)
p_result = urlparse(url)
proto = p_result.scheme
port = p_result.port
# scan all mgrs to generate deps and to get standbys too.
# assume that they are all on the same port as the active mgr.
for dd in self.mgr.cache.get_daemons_by_service('mgr'):
# we consider mgr a dep even if the dashboard is disabled
# in order to be consistent with _calc_daemon_deps().
deps.append(dd.name())
if not port:
continue
if dd.daemon_id == self.mgr.get_mgr_id():
continue
assert dd.hostname is not None
addr = self.mgr.inventory.get_addr(dd.hostname)
dashboard_urls.append(build_url(scheme=proto, host=addr, port=port))
context = {
'dashboard_urls': dashboard_urls,
'default_webhook_urls': default_webhook_urls
}
yml = self.mgr.template.render('services/alertmanager/alertmanager.yml.j2', context)
peers = []
port = 9094
for dd in self.mgr.cache.get_daemons_by_service('alertmanager'):
assert dd.hostname is not None
deps.append(dd.name())
addr = self.mgr.inventory.get_addr(dd.hostname)
peers.append(build_url(host=addr, port=port).lstrip('/'))
return {
"files": {
"alertmanager.yml": yml
},
"peers": peers
}, sorted(deps)
def request_begin(self):
req = cherrypy.request
user = JwtManager.get_username()
# Log the request.
self.logger.debug('[%s:%s] [%s] [%s] %s', req.remote.ip,
req.remote.port, req.method, user, req.path_info)
# Audit the request.
if Settings.AUDIT_API_ENABLED and req.method not in ['GET']:
url = build_url(req.remote.ip,
scheme=req.scheme,
port=req.remote.port)
msg = '[DASHBOARD] from=\'{}\' path=\'{}\' method=\'{}\' ' \
'user=\'{}\''.format(url, req.path_info, req.method, user)
if Settings.AUDIT_API_LOG_PAYLOAD:
params = dict(req.params or {}, **get_request_body_params(req))
# Hide sensitive data like passwords, secret keys, ...
# Extend the list of patterns to search for if necessary.
# Currently parameters like this are processed:
# - secret_key
# - user_password
# - new_passwd_to_login
keys = []
for key in ['password', 'passwd', 'secret']:
keys.extend([x for x in params.keys() if key in x])
for key in keys:
params[key] = '***'
msg = '{} params=\'{}\''.format(msg, json.dumps(params))
mgr.cluster_log('audit', mgr.ClusterLogPrio.INFO, msg)
def alertmgr_sd_config(self) -> List[Dict[str, Collection[str]]]:
"""Return <http_sd_config> compatible prometheus config for mgr alertmanager service."""
srv_entries = []
for dd in self.mgr.cache.get_daemons_by_service('alertmanager'):
assert dd.hostname is not None
addr = dd.ip if dd.ip else self.mgr.inventory.get_addr(dd.hostname)
port = dd.ports[0] if dd.ports else 9093
srv_entries.append('{}'.format(build_url(host=addr, port=port).lstrip('/')))
return [{"targets": srv_entries, "labels": {}}]
def config_dashboard(self, daemon_descrs: List[DaemonDescription]) -> None:
dd = self.get_active_daemon(daemon_descrs)
assert dd.hostname is not None
addr = dd.ip if dd.ip else self._inventory_get_fqdn(dd.hostname)
port = dd.ports[0] if dd.ports else self.DEFAULT_SERVICE_PORT
service_url = build_url(scheme='http', host=addr, port=port)
self._set_service_url_on_dashboard(
'Prometheus', 'dashboard get-prometheus-api-host',
'dashboard set-prometheus-api-host', service_url)
def config_dashboard(self, daemon_descrs: List[DaemonDescription]) -> None:
# TODO: signed cert
dd = self.get_active_daemon(daemon_descrs)
assert dd.hostname is not None
addr = dd.ip if dd.ip else self._inventory_get_addr(dd.hostname)
port = dd.ports[0] if dd.ports else self.DEFAULT_SERVICE_PORT
service_url = build_url(scheme='https', host=addr, port=port)
self._set_service_url_on_dashboard('Grafana',
'dashboard get-grafana-api-url',
'dashboard set-grafana-api-url',
service_url)
def node_exporter_sd_config(self) -> List[Dict[str, Collection[str]]]:
"""Return <http_sd_config> compatible prometheus config for node-exporter service."""
srv_entries = []
for dd in self.mgr.cache.get_daemons_by_service('node-exporter'):
assert dd.hostname is not None
addr = dd.ip if dd.ip else self.mgr.inventory.get_addr(dd.hostname)
port = dd.ports[0] if dd.ports else 9100
srv_entries.append({
'targets': [build_url(host=addr, port=port).lstrip('/')],
'labels': {'instance': dd.hostname}
})
return srv_entries
def prepare_create(
self,
daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonDeploySpec:
assert self.TYPE == daemon_spec.daemon_type
collectors = []
for dd in self.mgr.cache.get_daemons_by_type(
JaegerCollectorService.TYPE):
# scrape jaeger-collector nodes
assert dd.hostname is not None
port = dd.ports[
0] if dd.ports else JaegerCollectorService.DEFAULT_SERVICE_PORT
url = build_url(host=dd.hostname, port=port).lstrip('/')
collectors.append(url)
daemon_spec.final_config = {'collector_nodes': ",".join(collectors)}
return daemon_spec
def __init__(self,
host: str,
port: int,
client_name: Optional[str] = None,
ssl: bool = False,
auth: Optional[AuthBase] = None,
ssl_verify: bool = True) -> None:
super(RestClient, self).__init__()
self.client_name = client_name if client_name else ''
self.host = host
self.port = port
self.base_url = build_url(scheme='https' if ssl else 'http',
host=host,
port=port)
logger.debug("REST service base URL: %s", self.base_url)
self.headers = {'Accept': 'application/json'}
self.auth = auth
self.session = TimeoutRequestsSession()
self.session.verify = ssl_verify
def __init__(self,
access_key: str,
secret_key: str,
daemon_name: str,
user_id: Optional[str] = None) -> None:
try:
daemon = RgwClient._daemons[daemon_name]
except KeyError as error:
raise DashboardException(
msg='RGW Daemon not found: {}'.format(error),
http_status_code=404,
component='rgw')
ssl_verify = Settings.RGW_API_SSL_VERIFY
self.admin_path = Settings.RGW_API_ADMIN_RESOURCE
self.service_url = build_url(host=daemon.host, port=daemon.port)
self.auth = S3Auth(access_key,
secret_key,
service_url=self.service_url)
super(RgwClient, self).__init__(daemon.host,
daemon.port,
'RGW',
daemon.ssl,
self.auth,
ssl_verify=ssl_verify)
self.got_keys_from_config = not user_id
try:
self.userid = self._get_user_id(self.admin_path) if self.got_keys_from_config \
else user_id
except RequestException as error:
logger.exception(error)
msg = 'Error connecting to Object Gateway'
if error.status_code == 404:
msg = '{}: {}'.format(msg, str(error))
raise DashboardException(msg=msg,
http_status_code=error.status_code,
component='rgw')
self.daemon = daemon
logger.info(
"Created new connection: daemon=%s, host=%s, port=%s, ssl=%d, sslverify=%d",
daemon.name, daemon.host, daemon.port, daemon.ssl, ssl_verify)
def get_elasticsearch_nodes(service: CephadmService,
daemon_spec: CephadmDaemonDeploySpec) -> List[str]:
elasticsearch_nodes = []
for dd in service.mgr.cache.get_daemons_by_type(ElasticSearchService.TYPE):
assert dd.hostname is not None
addr = dd.ip if dd.ip else service.mgr.inventory.get_addr(dd.hostname)
port = dd.ports[
0] if dd.ports else ElasticSearchService.DEFAULT_SERVICE_PORT
url = build_url(host=addr, port=port).lstrip('/')
elasticsearch_nodes.append(f'http://{url}')
if len(elasticsearch_nodes) == 0:
# takes elasticsearch address from TracingSpec
spec: TracingSpec = cast(
TracingSpec,
service.mgr.spec_store.active_specs[daemon_spec.service_name])
assert spec.es_nodes is not None
urls = spec.es_nodes.split(",")
for url in urls:
elasticsearch_nodes.append(f'http://{url}')
return elasticsearch_nodes
def _serve(self):
# Load stored authentication keys
self.refresh_keys()
jsonify._instance = jsonify.GenericJSON(
sort_keys=True,
indent=4,
separators=(',', ': '),
)
server_addr = self.get_localized_module_option('server_addr', '::')
if server_addr is None:
raise CannotServe(
'no server_addr configured; try "ceph config-key set mgr/restful/server_addr <ip>"'
)
server_port = int(
self.get_localized_module_option('server_port', '8003'))
self.log.info('server_addr: %s server_port: %d', server_addr,
server_port)
cert = self.get_localized_store("crt")
if cert is not None:
cert_tmp = tempfile.NamedTemporaryFile()
cert_tmp.write(cert.encode('utf-8'))
cert_tmp.flush()
cert_fname = cert_tmp.name
else:
cert_fname = self.get_localized_store('crt_file')
pkey = self.get_localized_store("key")
if pkey is not None:
pkey_tmp = tempfile.NamedTemporaryFile()
pkey_tmp.write(pkey.encode('utf-8'))
pkey_tmp.flush()
pkey_fname = pkey_tmp.name
else:
pkey_fname = self.get_localized_module_option('key_file')
self.enable_auth = self.get_localized_module_option(
'enable_auth', True)
if not cert_fname or not pkey_fname:
raise CannotServe('no certificate configured')
if not os.path.isfile(cert_fname):
raise CannotServe('certificate %s does not exist' % cert_fname)
if not os.path.isfile(pkey_fname):
raise CannotServe('private key %s does not exist' % pkey_fname)
# Publish the URI that others may use to access the service we're
# about to start serving
addr = self.get_mgr_ip() if server_addr == "::" else server_addr
self.set_uri(build_url(scheme='https', host=addr, port=server_port))
# Create the HTTPS werkzeug server serving pecan app
self.server = make_server(
host=server_addr,
port=server_port,
app=make_app(
root='restful.api.Root',
hooks=[ErrorHook()], # use a callable if pecan >= 0.3.2
),
ssl_context=(cert_fname, pkey_fname),
)
sock_fd_flag = fcntl.fcntl(self.server.socket.fileno(), fcntl.F_GETFD)
if not (sock_fd_flag & fcntl.FD_CLOEXEC):
self.log.debug("set server socket close-on-exec")
fcntl.fcntl(self.server.socket.fileno(), fcntl.F_SETFD,
sock_fd_flag | fcntl.FD_CLOEXEC)
if self.stop_server:
self.log.debug('made server, but stop flag set')
else:
self.log.debug('made server, serving forever')
self.server.serve_forever()
def _configure(self):
"""
Configure CherryPy and initialize self.url_prefix
:returns our URI
"""
server_addr = self.get_localized_module_option( # type: ignore
'server_addr', get_default_addr())
use_ssl = self.get_localized_module_option('ssl', True) # type: ignore
if not use_ssl:
server_port = self.get_localized_module_option(
'server_port', 8080) # type: ignore
else:
server_port = self.get_localized_module_option(
'ssl_server_port', 8443) # type: ignore
if server_addr is None:
raise ServerConfigException(
'no server_addr configured; '
'try "ceph config set mgr mgr/{}/{}/server_addr <ip>"'.format(
self.module_name, self.get_mgr_id())) # type: ignore
self.log.info(
'server: ssl=%s host=%s port=%d',
'yes' if use_ssl else 'no', # type: ignore
server_addr,
server_port)
# Initialize custom handlers.
cherrypy.tools.authenticate = AuthManagerTool()
cherrypy.tools.plugin_hooks_filter_request = cherrypy.Tool(
'before_handler',
lambda: PLUGIN_MANAGER.hook.filter_request_before_handler(
request=cherrypy.request),
priority=1)
cherrypy.tools.request_logging = RequestLoggingTool()
cherrypy.tools.dashboard_exception_handler = HandlerWrapperTool(
dashboard_exception_handler, priority=31)
cherrypy.log.access_log.propagate = False
cherrypy.log.error_log.propagate = False
# Apply the 'global' CherryPy configuration.
config = {
'engine.autoreload.on':
False,
'server.socket_host':
server_addr,
'server.socket_port':
int(server_port),
'error_page.default':
json_error_page,
'tools.request_logging.on':
True,
'tools.gzip.on':
True,
'tools.gzip.mime_types': [
# text/html and text/plain are the default types to compress
'text/html',
'text/plain',
# We also want JSON and JavaScript to be compressed
'application/json',
'application/*+json',
'application/javascript',
],
'tools.json_in.on':
True,
'tools.json_in.force':
True,
'tools.plugin_hooks_filter_request.on':
True,
}
if use_ssl:
# SSL initialization
cert = self.get_localized_store("crt") # type: ignore
if cert is not None:
self.cert_tmp = tempfile.NamedTemporaryFile()
self.cert_tmp.write(cert.encode('utf-8'))
self.cert_tmp.flush() # cert_tmp must not be gc'ed
cert_fname = self.cert_tmp.name
else:
cert_fname = self.get_localized_module_option(
'crt_file') # type: ignore
pkey = self.get_localized_store("key") # type: ignore
if pkey is not None:
self.pkey_tmp = tempfile.NamedTemporaryFile()
self.pkey_tmp.write(pkey.encode('utf-8'))
self.pkey_tmp.flush() # pkey_tmp must not be gc'ed
pkey_fname = self.pkey_tmp.name
else:
pkey_fname = self.get_localized_module_option(
'key_file') # type: ignore
verify_tls_files(cert_fname, pkey_fname)
# Create custom SSL context to disable TLS 1.0 and 1.1.
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
context.load_cert_chain(cert_fname, pkey_fname)
if sys.version_info >= (3, 7):
context.minimum_version = ssl.TLSVersion.TLSv1_2
else:
context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
config['server.ssl_module'] = 'builtin'
config['server.ssl_certificate'] = cert_fname
config['server.ssl_private_key'] = pkey_fname
config['server.ssl_context'] = context
self.update_cherrypy_config(config)
self._url_prefix = prepare_url_prefix(
self.get_module_option( # type: ignore
'url_prefix', default=''))
if server_addr in ['::', '0.0.0.0']:
server_addr = self.get_mgr_ip() # type: ignore
base_url = build_url(
scheme='https' if use_ssl else 'http',
host=server_addr,
port=server_port,
)
uri = f'{base_url}{self.url_prefix}/'
return uri
def generate_config(
self,
daemon_spec: CephadmDaemonDeploySpec,
) -> Tuple[Dict[str, Any], List[str]]:
assert self.TYPE == daemon_spec.daemon_type
deps = [] # type: List[str]
# scrape mgrs
mgr_scrape_list = []
mgr_map = self.mgr.get('mgr_map')
port = cast(int, self.mgr.get_module_option_ex(
'prometheus', 'server_port', self.DEFAULT_MGR_PROMETHEUS_PORT))
deps.append(str(port))
t = mgr_map.get('services', {}).get('prometheus', None)
if t:
p_result = urlparse(t)
# urlparse .hostname removes '[]' from the hostname in case
# of ipv6 addresses so if this is the case then we just
# append the brackets when building the final scrape endpoint
if '[' in p_result.netloc and ']' in p_result.netloc:
mgr_scrape_list.append(f"[{p_result.hostname}]:{port}")
else:
mgr_scrape_list.append(f"{p_result.hostname}:{port}")
# scan all mgrs to generate deps and to get standbys too.
# assume that they are all on the same port as the active mgr.
for dd in self.mgr.cache.get_daemons_by_service('mgr'):
# we consider the mgr a dep even if the prometheus module is
# disabled in order to be consistent with _calc_daemon_deps().
deps.append(dd.name())
if not port:
continue
if dd.daemon_id == self.mgr.get_mgr_id():
continue
assert dd.hostname is not None
addr = self._inventory_get_fqdn(dd.hostname)
mgr_scrape_list.append(build_url(host=addr, port=port).lstrip('/'))
# scrape node exporters
nodes = []
for dd in self.mgr.cache.get_daemons_by_service('node-exporter'):
assert dd.hostname is not None
deps.append(dd.name())
addr = dd.ip if dd.ip else self._inventory_get_fqdn(dd.hostname)
port = dd.ports[0] if dd.ports else 9100
nodes.append({
'hostname': dd.hostname,
'url': build_url(host=addr, port=port).lstrip('/')
})
# scrape alert managers
alertmgr_targets = []
for dd in self.mgr.cache.get_daemons_by_service('alertmanager'):
assert dd.hostname is not None
deps.append(dd.name())
addr = dd.ip if dd.ip else self._inventory_get_fqdn(dd.hostname)
port = dd.ports[0] if dd.ports else 9093
alertmgr_targets.append("'{}'".format(build_url(host=addr, port=port).lstrip('/')))
# scrape haproxies
haproxy_targets = []
for dd in self.mgr.cache.get_daemons_by_type('ingress'):
if dd.service_name() in self.mgr.spec_store:
spec = cast(IngressSpec, self.mgr.spec_store[dd.service_name()].spec)
assert dd.hostname is not None
deps.append(dd.name())
if dd.daemon_type == 'haproxy':
addr = self._inventory_get_fqdn(dd.hostname)
haproxy_targets.append({
"url": f"'{build_url(host=addr, port=spec.monitor_port).lstrip('/')}'",
"service": dd.service_name(),
})
# generate the prometheus configuration
context = {
'alertmgr_targets': alertmgr_targets,
'mgr_scrape_list': mgr_scrape_list,
'haproxy_targets': haproxy_targets,
'nodes': nodes,
}
r = {
'files': {
'prometheus.yml':
self.mgr.template.render(
'services/prometheus/prometheus.yml.j2', context)
}
}
# include alerts, if present in the container
if os.path.exists(self.mgr.prometheus_alerts_path):
with open(self.mgr.prometheus_alerts_path, 'r', encoding='utf-8') as f:
alerts = f.read()
r['files']['/etc/prometheus/alerting/ceph_alerts.yml'] = alerts
return r, sorted(deps)
def generate_config(
self, daemon_spec: CephadmDaemonDeploySpec
) -> Tuple[Dict[str, Any], List[str]]:
assert self.TYPE == daemon_spec.daemon_type
deps: List[str] = []
default_webhook_urls: List[str] = []
spec = cast(AlertManagerSpec,
self.mgr.spec_store[daemon_spec.service_name].spec)
try:
secure = spec.secure
except AttributeError:
secure = False
user_data = spec.user_data
if 'default_webhook_urls' in user_data and isinstance(
user_data['default_webhook_urls'], list):
default_webhook_urls.extend(user_data['default_webhook_urls'])
# dashboard(s)
dashboard_urls: List[str] = []
snmp_gateway_urls: List[str] = []
mgr_map = self.mgr.get('mgr_map')
port = None
proto = None # http: or https:
url = mgr_map.get('services', {}).get('dashboard', None)
if url:
p_result = urlparse(url.rstrip('/'))
hostname = socket.getfqdn(p_result.hostname)
try:
ip = ipaddress.ip_address(hostname)
except ValueError:
pass
else:
if ip.version == 6:
hostname = f'[{hostname}]'
dashboard_urls.append(
f'{p_result.scheme}://{hostname}:{p_result.port}{p_result.path}'
)
proto = p_result.scheme
port = p_result.port
# scan all mgrs to generate deps and to get standbys too.
# assume that they are all on the same port as the active mgr.
for dd in self.mgr.cache.get_daemons_by_service('mgr'):
# we consider mgr a dep even if the dashboard is disabled
# in order to be consistent with _calc_daemon_deps().
deps.append(dd.name())
if not port:
continue
if dd.daemon_id == self.mgr.get_mgr_id():
continue
assert dd.hostname is not None
addr = self._inventory_get_fqdn(dd.hostname)
dashboard_urls.append(
build_url(scheme=proto, host=addr, port=port).rstrip('/'))
for dd in self.mgr.cache.get_daemons_by_service('snmp-gateway'):
assert dd.hostname is not None
assert dd.ports
addr = dd.ip if dd.ip else self._inventory_get_fqdn(dd.hostname)
deps.append(dd.name())
snmp_gateway_urls.append(
build_url(scheme='http',
host=addr,
port=dd.ports[0],
path='/alerts'))
context = {
'dashboard_urls': dashboard_urls,
'default_webhook_urls': default_webhook_urls,
'snmp_gateway_urls': snmp_gateway_urls,
'secure': secure,
}
yml = self.mgr.template.render(
'services/alertmanager/alertmanager.yml.j2', context)
peers = []
port = 9094
for dd in self.mgr.cache.get_daemons_by_service('alertmanager'):
assert dd.hostname is not None
deps.append(dd.name())
addr = self._inventory_get_fqdn(dd.hostname)
peers.append(build_url(host=addr, port=port).lstrip('/'))
return {
"files": {
"alertmanager.yml": yml
},
"peers": peers
}, sorted(deps)
