def test_monitoring_ports(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
yaml_str = """service_type: alertmanager
service_name: alertmanager
placement:
count: 1
spec:
port: 4200
"""
yaml_file = yaml.safe_load(yaml_str)
spec = ServiceSpec.from_json(yaml_file)
with patch("cephadm.services.monitoring.AlertmanagerService.generate_config", return_value=({}, [])):
with with_service(cephadm_module, spec):
CephadmServe(cephadm_module)._check_daemons()
_run_cephadm.assert_called_with(
'test', 'alertmanager.test', 'deploy', [
'--name', 'alertmanager.test',
'--meta-json', '{"service_name": "alertmanager", "ports": [4200, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null}',
'--config-json', '-',
'--tcp-ports', '4200 9094',
'--reconfig'
],
stdin='{}',
image='')
def test_snmp_v2c_with_port(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
spec = SNMPGatewaySpec(
snmp_version='V2c',
snmp_destination='192.168.1.1:162',
credentials={
'snmp_community': 'public'
},
port=9465)
config = {
"destination": spec.snmp_destination,
"snmp_version": spec.snmp_version,
"snmp_community": spec.credentials.get('snmp_community')
}
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, spec):
_run_cephadm.assert_called_with(
'test',
'snmp-gateway.test',
'deploy',
[
'--name', 'snmp-gateway.test',
'--meta-json',
'{"service_name": "snmp-gateway", "ports": [9465], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null}',
'--config-json', '-',
'--tcp-ports', '9465'
],
stdin=json.dumps(config),
image=''
)
def test_snmp_v3nopriv_deployment(self, _run_cephadm,
cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
spec = SNMPGatewaySpec(snmp_version='V3',
snmp_destination='192.168.1.1:162',
engine_id='8000C53F00000000',
credentials={
'snmp_v3_auth_username': '******',
'snmp_v3_auth_password': '******'
})
config = {
'destination': spec.snmp_destination,
'snmp_version': spec.snmp_version,
'snmp_v3_auth_protocol': 'SHA',
'snmp_v3_auth_username': '******',
'snmp_v3_auth_password': '******',
'snmp_v3_engine_id': '8000C53F00000000'
}
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, spec):
_run_cephadm.assert_called_with(
'test',
'snmp-gateway.test',
'deploy', [
'--name', 'snmp-gateway.test', '--meta-json',
'{"service_name": "snmp-gateway", "ports": [9464], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
'--config-json', '-', '--tcp-ports', '9464'
],
stdin=json.dumps(config),
image='')
def test_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, ServiceSpec('cephfs-mirror')):
cephadm_module.assert_issued_mon_command({
'prefix': 'mgr module enable',
'module': 'mirroring'
})
def test_offline(self, check_execute_command, execute_command, cephadm_module):
check_execute_command.side_effect = async_side_effect('')
execute_command.side_effect = async_side_effect(('', '', 0))
if not AsyncMock:
# can't run this test if we could not import AsyncMock
return
mock_connect = AsyncMock(return_value='')
with mock.patch("asyncssh.connect", new=mock_connect) as asyncssh_connect:
with with_host(cephadm_module, 'test'):
asyncssh_connect.side_effect = ConnectionLost('reason')
code, out, err = cephadm_module.check_host('test')
assert out == ''
assert "Host 'test' not found" in err
out = wait(cephadm_module, cephadm_module.get_hosts())[0].to_json()
assert out == HostSpec('test', '1::4', status='Offline').to_json()
asyncssh_connect.return_value = mock.MagicMock()
asyncssh_connect.side_effect = None
assert CephadmServe(cephadm_module)._check_host('test') is None
out = wait(cephadm_module, cephadm_module.get_hosts())[0].to_json()
assert out == HostSpec('test', '1::4').to_json()
def test_loki_config(self, _run_cephadm,
cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, MonitoringSpec('loki')) as _:
y = dedent("""
# This file is generated by cephadm.
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 8080
common:
path_prefix: /tmp/loki
storage:
filesystem:
chunks_directory: /tmp/loki/chunks
rules_directory: /tmp/loki/rules
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h""").lstrip()
_run_cephadm.assert_called_with(
'test',
'loki.test',
'deploy', [
'--name', 'loki.test', '--meta-json',
'{"service_name": "loki", "ports": [3100], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
'--config-json', '-', '--tcp-ports', '3100'
],
stdin=json.dumps({"files": {
"loki.yml": y
}}),
image='')
def test_alertmanager_config(self, _run_cephadm,
cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, AlertManagerSpec()):
y = dedent("""
# This file is generated by cephadm.
# See https://prometheus.io/docs/alerting/configuration/ for documentation.
global:
resolve_timeout: 5m
http_config:
tls_config:
insecure_skip_verify: true
route:
receiver: 'default'
routes:
- group_by: ['alertname']
group_wait: 10s
group_interval: 10s
repeat_interval: 1h
receiver: 'ceph-dashboard'
receivers:
- name: 'default'
webhook_configs:
- name: 'ceph-dashboard'
webhook_configs:
- url: 'http://[::1]:8080/api/prometheus_receiver'
""").lstrip()
_run_cephadm.assert_called_with(
'test',
'alertmanager.test',
'deploy',
[
'--name', 'alertmanager.test',
'--meta-json', '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
'--config-json', '-', '--tcp-ports', '9093 9094'
],
stdin=json.dumps({"files": {"alertmanager.yml": y}, "peers": []}),
image='')\
def test_prometheus_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, MonitoringSpec('node-exporter')) as _, \
with_service(cephadm_module, MonitoringSpec('prometheus')) as _:
y = dedent("""
# This file is generated by cephadm.
global:
scrape_interval: 10s
evaluation_interval: 10s
rule_files:
- /etc/prometheus/alerting/*
scrape_configs:
- job_name: 'ceph'
honor_labels: true
static_configs:
- targets:
- '[::1]:8081'
- job_name: 'node'
static_configs:
- targets: ['[1::4]:9100']
labels:
instance: 'test'
""").lstrip()
_run_cephadm.assert_called_with(
'test',
'prometheus.test',
'deploy',
[
'--name', 'prometheus.test',
'--meta-json',
'{"service_name": "prometheus", "ports": [9095], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null}',
'--config-json', '-',
'--tcp-ports', '9095'
],
stdin=json.dumps({"files": {"prometheus.yml": y}}),
image='')
def test_promtail_config(self, _run_cephadm,
cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, ServiceSpec('mgr')) as _, \
with_service(cephadm_module, MonitoringSpec('promtail')) as _:
y = dedent("""
# This file is generated by cephadm.
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://1::4:3100/loki/api/v1/push
scrape_configs:
- job_name: system
static_configs:
- targets:
- 1::4
labels:
job: Cluster Logs
__path__: /var/log/ceph/**/*.log""").lstrip()
_run_cephadm.assert_called_with(
'test',
'promtail.test',
'deploy', [
'--name', 'promtail.test', '--meta-json',
'{"service_name": "promtail", "ports": [9080], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
'--config-json', '-', '--tcp-ports', '9080'
],
stdin=json.dumps({"files": {
"promtail.yml": y
}}),
image='')
def test_grafana_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
cephadm_module.set_store('grafana_crt', 'c')
cephadm_module.set_store('grafana_key', 'k')
with with_service(cephadm_module, MonitoringSpec('prometheus')) as _, \
with_service(cephadm_module, GrafanaSpec('grafana')) as _:
files = {
'grafana.ini': dedent("""
# This file is generated by cephadm.
[users]
default_theme = light
[auth.anonymous]
enabled = true
org_name = 'Main Org.'
org_role = 'Viewer'
[server]
domain = 'bootstrap.storage.lab'
protocol = https
cert_file = /etc/grafana/certs/cert_file
cert_key = /etc/grafana/certs/cert_key
http_port = 3000
http_addr =
[security]
disable_initial_admin_creation = true
cookie_secure = true
cookie_samesite = none
allow_embedding = true""").lstrip(), # noqa: W291
'provisioning/datasources/ceph-dashboard.yml': dedent("""
# This file is generated by cephadm.
deleteDatasources:
- name: 'Dashboard1'
orgId: 1
datasources:
- name: 'Dashboard1'
type: 'prometheus'
access: 'proxy'
orgId: 1
url: 'http://[1::4]:9095'
basicAuth: false
isDefault: true
editable: false
""").lstrip(),
'certs/cert_file': dedent("""
# generated by cephadm
c""").lstrip(),
'certs/cert_key': dedent("""
# generated by cephadm
k""").lstrip(),
}
_run_cephadm.assert_called_with(
'test',
'grafana.test',
'deploy',
[
'--name', 'grafana.test',
'--meta-json',
'{"service_name": "grafana", "ports": [3000], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null}',
'--config-json', '-', '--tcp-ports', '3000'],
stdin=json.dumps({"files": files}),
image='')
def test_ingress_config(self, _run_cephadm,
cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
cephadm_module.cache.update_host_networks(
'test', {'1.2.3.0/24': {
'if0': ['1.2.3.4/32']
}})
# the ingress backend
s = RGWSpec(service_id="foo",
placement=PlacementSpec(count=1),
rgw_frontend_type='beast')
ispec = IngressSpec(service_type='ingress',
service_id='test',
backend_service='rgw.foo',
frontend_port=8089,
monitor_port=8999,
monitor_user='******',
monitor_password='******',
keepalived_password='******',
virtual_interface_networks=['1.2.3.0/24'],
virtual_ip="1.2.3.4/32")
with with_service(cephadm_module,
s) as _, with_service(cephadm_module,
ispec) as _:
# generate the keepalived conf based on the specified spec
keepalived_generated_conf = cephadm_module.cephadm_services[
'ingress'].keepalived_generate_config(
CephadmDaemonDeploySpec(
host='test',
daemon_id='ingress',
service_name=ispec.service_name()))
keepalived_expected_conf = {
'files': {
'keepalived.conf':
'# This file is generated by cephadm.\n'
'vrrp_script check_backend {\n '
'script "/usr/bin/curl http://localhost:8999/health"\n '
'weight -20\n '
'interval 2\n '
'rise 2\n '
'fall 2\n}\n\n'
'vrrp_instance VI_0 {\n '
'state MASTER\n '
'priority 100\n '
'interface if0\n '
'virtual_router_id 51\n '
'advert_int 1\n '
'authentication {\n '
'auth_type PASS\n '
'auth_pass 12345\n '
'}\n '
'unicast_src_ip 1::4\n '
'unicast_peer {\n '
'}\n '
'virtual_ipaddress {\n '
'1.2.3.4/32 dev if0\n '
'}\n '
'track_script {\n '
'check_backend\n }\n'
'}'
}
}
# check keepalived config
assert keepalived_generated_conf[0] == keepalived_expected_conf
# generate the haproxy conf based on the specified spec
haproxy_generated_conf = cephadm_module.cephadm_services[
'ingress'].haproxy_generate_config(
CephadmDaemonDeploySpec(
host='test',
daemon_id='ingress',
service_name=ispec.service_name()))
haproxy_expected_conf = {
'files': {
'haproxy.cfg':
'# This file is generated by cephadm.'
'\nglobal\n log '
'127.0.0.1 local2\n '
'chroot /var/lib/haproxy\n '
'pidfile /var/lib/haproxy/haproxy.pid\n '
'maxconn 8000\n '
'daemon\n '
'stats socket /var/lib/haproxy/stats\n'
'\ndefaults\n '
'mode http\n '
'log global\n '
'option httplog\n '
'option dontlognull\n '
'option http-server-close\n '
'option forwardfor except 127.0.0.0/8\n '
'option redispatch\n '
'retries 3\n '
'timeout queue 20s\n '
'timeout connect 5s\n '
'timeout http-request 1s\n '
'timeout http-keep-alive 5s\n '
'timeout client 1s\n '
'timeout server 1s\n '
'timeout check 5s\n '
'maxconn 8000\n'
'\nfrontend stats\n '
'mode http\n '
'bind 1.2.3.4:8999\n '
'bind localhost:8999\n '
'stats enable\n '
'stats uri /stats\n '
'stats refresh 10s\n '
'stats auth admin:12345\n '
'http-request use-service prometheus-exporter if { path /metrics }\n '
'monitor-uri /health\n'
'\nfrontend frontend\n '
'bind 1.2.3.4:8089\n '
'default_backend backend\n\n'
'backend backend\n '
'option forwardfor\n '
'balance static-rr\n '
'option httpchk HEAD / HTTP/1.0\n '
'server ' + haproxy_generated_conf[1][0] +
' 1::4:80 check weight 100\n'
}
}
assert haproxy_generated_conf[0] == haproxy_expected_conf