def test_migrate_scheduler(cephadm_module: CephadmOrchestrator):
with with_host(cephadm_module, 'host1'):
with with_host(cephadm_module, 'host2'):
# emulate the old scheduler:
c = cephadm_module.apply_rgw(
ServiceSpec('rgw',
'r.z',
placement=PlacementSpec(host_pattern='*',
count=2)))
assert wait(cephadm_module, c) == 'Scheduled rgw.r.z update...'
cephadm_module._apply_all_services()
out = {
o.hostname
for o in wait(cephadm_module, cephadm_module.list_daemons())
}
assert out == {'host1', 'host2'}
c = cephadm_module.apply_rgw(
ServiceSpec('rgw',
'r.z',
placement=PlacementSpec(host_pattern='host1',
count=2)))
assert wait(cephadm_module, c) == 'Scheduled rgw.r.z update...'
cephadm_module.migration_current = 0
cephadm_module.migration.migrate()
# assert we need all daemons.
assert cephadm_module.migration_current == 0
# Sorry, for this hack, but I need to make sure, Migration thinks,
# we have updated all daemons already.
cephadm_module.cache.last_daemon_update['host1'] = datetime.now()
cephadm_module.cache.last_daemon_update['host2'] = datetime.now()
cephadm_module.migration.migrate()
assert cephadm_module.migration_current == 2
out = [
o.spec.placement for o in wait(
cephadm_module, cephadm_module.describe_service())
]
assert out == [
PlacementSpec(count=2,
hosts=[
HostPlacementSpec(hostname='host1',
network='',
name=''),
HostPlacementSpec(hostname='host2',
network='',
name='')
])
]
def test_snmp_v2c_with_port(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
spec = SNMPGatewaySpec(
snmp_version='V2c',
snmp_destination='192.168.1.1:162',
credentials={
'snmp_community': 'public'
},
port=9465)
config = {
"destination": spec.snmp_destination,
"snmp_version": spec.snmp_version,
"snmp_community": spec.credentials.get('snmp_community')
}
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, spec):
_run_cephadm.assert_called_with(
'test',
'snmp-gateway.test',
'deploy',
[
'--name', 'snmp-gateway.test',
'--meta-json',
'{"service_name": "snmp-gateway", "ports": [9465], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null}',
'--config-json', '-',
'--tcp-ports', '9465'
],
stdin=json.dumps(config),
image=''
)
def test_snmp_v3nopriv_deployment(self, _run_cephadm,
cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
spec = SNMPGatewaySpec(snmp_version='V3',
snmp_destination='192.168.1.1:162',
engine_id='8000C53F00000000',
credentials={
'snmp_v3_auth_username': '******',
'snmp_v3_auth_password': '******'
})
config = {
'destination': spec.snmp_destination,
'snmp_version': spec.snmp_version,
'snmp_v3_auth_protocol': 'SHA',
'snmp_v3_auth_username': '******',
'snmp_v3_auth_password': '******',
'snmp_v3_engine_id': '8000C53F00000000'
}
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, spec):
_run_cephadm.assert_called_with(
'test',
'snmp-gateway.test',
'deploy', [
'--name', 'snmp-gateway.test', '--meta-json',
'{"service_name": "snmp-gateway", "ports": [9464], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
'--config-json', '-', '--tcp-ports', '9464'
],
stdin=json.dumps(config),
image='')
def test_migrate_service_id_mon_one(cephadm_module: CephadmOrchestrator):
with with_host(cephadm_module, 'host1'):
cephadm_module.set_store(
SPEC_STORE_PREFIX + 'mon.wrong',
json.dumps(
{
'spec': {
'service_type': 'mon',
'service_id': 'wrong',
'placement': {
'hosts': ['host1']
}
},
'created': datetime_to_str(datetime_now()),
},
sort_keys=True),
)
cephadm_module.spec_store.load()
assert len(cephadm_module.spec_store.specs) == 1
assert cephadm_module.spec_store.specs['mon.wrong'].service_name(
) == 'mon'
cephadm_module.migration_current = 1
cephadm_module.migration.migrate()
assert cephadm_module.migration_current == 2
assert len(cephadm_module.spec_store.specs) == 1
assert cephadm_module.spec_store.specs['mon'] == ServiceSpec(
service_type='mon',
unmanaged=True,
placement=PlacementSpec(hosts=['host1']))
def test_migrate_nfs_initial_octopus(cephadm_module: CephadmOrchestrator):
with with_host(cephadm_module, 'host1'):
cephadm_module.set_store(
SPEC_STORE_PREFIX + 'mds',
json.dumps({
'spec': {
'service_type': 'nfs',
'service_id': 'ganesha-foo',
'placement': {
'hosts': ['host1']
},
'spec': {
'pool': 'mypool',
'namespace': 'foons',
},
},
'created': datetime_to_str(datetime_now()),
}, sort_keys=True),
)
cephadm_module.migration_current = 1
cephadm_module.spec_store.load()
ls = json.loads(cephadm_module.get_store('nfs_migration_queue'))
assert ls == [['ganesha-foo', 'mypool', 'foons']]
cephadm_module.migration.migrate(True)
assert cephadm_module.migration_current == 2
cephadm_module.migration.migrate()
assert cephadm_module.migration_current == 3
def test_monitoring_ports(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
yaml_str = """service_type: alertmanager
service_name: alertmanager
placement:
count: 1
spec:
port: 4200
"""
yaml_file = yaml.safe_load(yaml_str)
spec = ServiceSpec.from_json(yaml_file)
with patch("cephadm.services.monitoring.AlertmanagerService.generate_config", return_value=({}, [])):
with with_service(cephadm_module, spec):
CephadmServe(cephadm_module)._check_daemons()
_run_cephadm.assert_called_with(
'test', 'alertmanager.test', 'deploy', [
'--name', 'alertmanager.test',
'--meta-json', '{"service_name": "alertmanager", "ports": [4200, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null}',
'--config-json', '-',
'--tcp-ports', '4200 9094',
'--reconfig'
],
stdin='{}',
image='')
def test_offline(self, check_execute_command, execute_command,
cephadm_module):
check_execute_command.return_value = ''
execute_command.return_value = '', '', 0
if not AsyncMock:
# can't run this test if we could not import AsyncMock
return
mock_connect = AsyncMock(return_value='')
with mock.patch("asyncssh.connect",
new=mock_connect) as asyncssh_connect:
with with_host(cephadm_module, 'test'):
asyncssh_connect.side_effect = ConnectionLost('reason')
code, out, err = cephadm_module.check_host('test')
assert out == ''
assert "Host 'test' not found" in err
out = wait(cephadm_module,
cephadm_module.get_hosts())[0].to_json()
assert out == HostSpec('test', '1::4',
status='Offline').to_json()
asyncssh_connect.return_value = mock.MagicMock()
asyncssh_connect.side_effect = None
assert CephadmServe(cephadm_module)._check_host('test') is None
out = wait(cephadm_module,
cephadm_module.get_hosts())[0].to_json()
assert out == HostSpec('test', '1::4').to_json()
def test_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, ServiceSpec('cephfs-mirror')):
cephadm_module.assert_issued_mon_command({
'prefix': 'mgr module enable',
'module': 'mirroring'
})
def test_migrate_scheduler(cephadm_module: CephadmOrchestrator):
with with_host(cephadm_module, 'host1', refresh_hosts=False):
with with_host(cephadm_module, 'host2', refresh_hosts=False):
# emulate the old scheduler:
c = cephadm_module.apply_rgw(
ServiceSpec('rgw', 'r.z', placement=PlacementSpec(host_pattern='*', count=2))
)
assert wait(cephadm_module, c) == 'Scheduled rgw.r.z update...'
# with pytest.raises(OrchestratorError, match="cephadm migration still ongoing. Please wait, until the migration is complete."):
CephadmServe(cephadm_module)._apply_all_services()
cephadm_module.migration_current = 0
cephadm_module.migration.migrate()
# assert we need all daemons.
assert cephadm_module.migration_current == 0
CephadmServe(cephadm_module)._refresh_hosts_and_daemons()
receive_agent_metadata_all_hosts(cephadm_module)
cephadm_module.migration.migrate()
CephadmServe(cephadm_module)._apply_all_services()
out = {o.hostname for o in wait(cephadm_module, cephadm_module.list_daemons())}
assert out == {'host1', 'host2'}
c = cephadm_module.apply_rgw(
ServiceSpec('rgw', 'r.z', placement=PlacementSpec(host_pattern='host1', count=2))
)
assert wait(cephadm_module, c) == 'Scheduled rgw.r.z update...'
# Sorry, for this hack, but I need to make sure, Migration thinks,
# we have updated all daemons already.
cephadm_module.cache.last_daemon_update['host1'] = datetime_now()
cephadm_module.cache.last_daemon_update['host2'] = datetime_now()
cephadm_module.migration_current = 0
cephadm_module.migration.migrate()
assert cephadm_module.migration_current >= 2
out = [o.spec.placement for o in wait(
cephadm_module, cephadm_module.describe_service())]
assert out == [PlacementSpec(count=2, hosts=[HostPlacementSpec(
hostname='host1', network='', name=''), HostPlacementSpec(hostname='host2', network='', name='')])]
def test_grafana_initial_admin_pw(self,
cephadm_module: CephadmOrchestrator):
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, ServiceSpec('mgr')) as _, \
with_service(cephadm_module, GrafanaSpec(initial_admin_password='******')):
out = cephadm_module.cephadm_services[
'grafana'].generate_config(
CephadmDaemonDeploySpec('test', 'daemon', 'grafana'))
assert out == ({
'files': {
'grafana.ini':
'# This file is generated by cephadm.\n'
'[users]\n'
' default_theme = light\n'
'[auth.anonymous]\n'
' enabled = true\n'
" org_name = 'Main Org.'\n"
" org_role = 'Viewer'\n"
'[server]\n'
" domain = 'bootstrap.storage.lab'\n"
' protocol = https\n'
' cert_file = /etc/grafana/certs/cert_file\n'
' cert_key = /etc/grafana/certs/cert_key\n'
' http_port = 3000\n'
' http_addr = \n'
'[security]\n'
' admin_user = admin\n'
' admin_password = secure\n'
' cookie_secure = true\n'
' cookie_samesite = none\n'
' allow_embedding = true',
'provisioning/datasources/ceph-dashboard.yml':
"# This file is generated by cephadm.\n"
'deleteDatasources:\n\n'
" - name: 'Loki'\n"
' orgId: 2\n\n'
'datasources:\n\n'
" - name: 'Loki'\n"
" type: 'loki'\n"
" access: 'proxy'\n"
' orgId: 2\n'
" url: 'http://[1::4]:3100'\n"
' basicAuth: false\n'
' isDefault: true\n'
' editable: false',
'certs/cert_file':
ANY,
'certs/cert_key':
ANY
}
}, [])
def test_loki_config(self, _run_cephadm,
cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, MonitoringSpec('loki')) as _:
y = dedent("""
# This file is generated by cephadm.
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 8080
common:
path_prefix: /tmp/loki
storage:
filesystem:
chunks_directory: /tmp/loki/chunks
rules_directory: /tmp/loki/rules
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h""").lstrip()
_run_cephadm.assert_called_with(
'test',
'loki.test',
'deploy', [
'--name', 'loki.test', '--meta-json',
'{"service_name": "loki", "ports": [3100], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
'--config-json', '-', '--tcp-ports', '3100'
],
stdin=json.dumps({"files": {
"loki.yml": y
}}),
image='')
def test_migrate_service_id_mds_one(cephadm_module: CephadmOrchestrator):
with with_host(cephadm_module, 'host1'):
cephadm_module.set_store(SPEC_STORE_PREFIX + 'mds', json.dumps({
'spec': {
'service_type': 'mds',
'placement': {
'hosts': ['host1']
}
},
'created': datetime_to_str(datetime_now()),
}, sort_keys=True),
)
cephadm_module.spec_store.load()
# there is nothing to migrate, as the spec is gone now.
assert len(cephadm_module.spec_store.all_specs) == 0
def test_alertmanager_config(self, _run_cephadm,
cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, AlertManagerSpec()):
y = dedent("""
# This file is generated by cephadm.
# See https://prometheus.io/docs/alerting/configuration/ for documentation.
global:
resolve_timeout: 5m
http_config:
tls_config:
insecure_skip_verify: true
route:
receiver: 'default'
routes:
- group_by: ['alertname']
group_wait: 10s
group_interval: 10s
repeat_interval: 1h
receiver: 'ceph-dashboard'
receivers:
- name: 'default'
webhook_configs:
- name: 'ceph-dashboard'
webhook_configs:
- url: 'http://[::1]:8080/api/prometheus_receiver'
""").lstrip()
_run_cephadm.assert_called_with(
'test',
'alertmanager.test',
'deploy',
[
'--name', 'alertmanager.test',
'--meta-json', '{"service_name": "alertmanager", "ports": [9093, 9094], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
'--config-json', '-', '--tcp-ports', '9093 9094'
],
stdin=json.dumps({"files": {"alertmanager.yml": y}, "peers": []}),
image='')\
def test_rgw_update(self, frontend, ssl, expected, cephadm_module: CephadmOrchestrator):
with with_host(cephadm_module, 'host1'):
cephadm_module.cache.update_host_networks('host1', {
'fd00:fd00:fd00:3000::/64': {
'if0': ['fd00:fd00:fd00:3000::1']
}
})
s = RGWSpec(service_id="foo",
networks=['fd00:fd00:fd00:3000::/64'],
ssl=ssl,
rgw_frontend_type=frontend)
with with_service(cephadm_module, s) as dds:
_, f, _ = cephadm_module.check_mon_command({
'prefix': 'config get',
'who': f'client.{dds[0]}',
'key': 'rgw_frontends',
})
assert f == expected
def test_prometheus_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, MonitoringSpec('node-exporter')) as _, \
with_service(cephadm_module, MonitoringSpec('prometheus')) as _:
y = dedent("""
# This file is generated by cephadm.
global:
scrape_interval: 10s
evaluation_interval: 10s
rule_files:
- /etc/prometheus/alerting/*
scrape_configs:
- job_name: 'ceph'
honor_labels: true
static_configs:
- targets:
- '[::1]:8081'
- job_name: 'node'
static_configs:
- targets: ['[1::4]:9100']
labels:
instance: 'test'
""").lstrip()
_run_cephadm.assert_called_with(
'test',
'prometheus.test',
'deploy',
[
'--name', 'prometheus.test',
'--meta-json',
'{"service_name": "prometheus", "ports": [9095], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null}',
'--config-json', '-',
'--tcp-ports', '9095'
],
stdin=json.dumps({"files": {"prometheus.yml": y}}),
image='')
def test_promtail_config(self, _run_cephadm,
cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
with with_service(cephadm_module, ServiceSpec('mgr')) as _, \
with_service(cephadm_module, MonitoringSpec('promtail')) as _:
y = dedent("""
# This file is generated by cephadm.
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://1::4:3100/loki/api/v1/push
scrape_configs:
- job_name: system
static_configs:
- targets:
- 1::4
labels:
job: Cluster Logs
__path__: /var/log/ceph/**/*.log""").lstrip()
_run_cephadm.assert_called_with(
'test',
'promtail.test',
'deploy', [
'--name', 'promtail.test', '--meta-json',
'{"service_name": "promtail", "ports": [9080], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null, "extra_container_args": null}',
'--config-json', '-', '--tcp-ports', '9080'
],
stdin=json.dumps({"files": {
"promtail.yml": y
}}),
image='')
def test_ingress_config(self, _run_cephadm,
cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
cephadm_module.cache.update_host_networks(
'test', {'1.2.3.0/24': {
'if0': ['1.2.3.4/32']
}})
# the ingress backend
s = RGWSpec(service_id="foo",
placement=PlacementSpec(count=1),
rgw_frontend_type='beast')
ispec = IngressSpec(service_type='ingress',
service_id='test',
backend_service='rgw.foo',
frontend_port=8089,
monitor_port=8999,
monitor_user='******',
monitor_password='******',
keepalived_password='******',
virtual_interface_networks=['1.2.3.0/24'],
virtual_ip="1.2.3.4/32")
with with_service(cephadm_module,
s) as _, with_service(cephadm_module,
ispec) as _:
# generate the keepalived conf based on the specified spec
keepalived_generated_conf = cephadm_module.cephadm_services[
'ingress'].keepalived_generate_config(
CephadmDaemonDeploySpec(
host='test',
daemon_id='ingress',
service_name=ispec.service_name()))
keepalived_expected_conf = {
'files': {
'keepalived.conf':
'# This file is generated by cephadm.\n'
'vrrp_script check_backend {\n '
'script "/usr/bin/curl http://localhost:8999/health"\n '
'weight -20\n '
'interval 2\n '
'rise 2\n '
'fall 2\n}\n\n'
'vrrp_instance VI_0 {\n '
'state MASTER\n '
'priority 100\n '
'interface if0\n '
'virtual_router_id 51\n '
'advert_int 1\n '
'authentication {\n '
'auth_type PASS\n '
'auth_pass 12345\n '
'}\n '
'unicast_src_ip 1::4\n '
'unicast_peer {\n '
'}\n '
'virtual_ipaddress {\n '
'1.2.3.4/32 dev if0\n '
'}\n '
'track_script {\n '
'check_backend\n }\n'
'}'
}
}
# check keepalived config
assert keepalived_generated_conf[0] == keepalived_expected_conf
# generate the haproxy conf based on the specified spec
haproxy_generated_conf = cephadm_module.cephadm_services[
'ingress'].haproxy_generate_config(
CephadmDaemonDeploySpec(
host='test',
daemon_id='ingress',
service_name=ispec.service_name()))
haproxy_expected_conf = {
'files': {
'haproxy.cfg':
'# This file is generated by cephadm.'
'\nglobal\n log '
'127.0.0.1 local2\n '
'chroot /var/lib/haproxy\n '
'pidfile /var/lib/haproxy/haproxy.pid\n '
'maxconn 8000\n '
'daemon\n '
'stats socket /var/lib/haproxy/stats\n'
'\ndefaults\n '
'mode http\n '
'log global\n '
'option httplog\n '
'option dontlognull\n '
'option http-server-close\n '
'option forwardfor except 127.0.0.0/8\n '
'option redispatch\n '
'retries 3\n '
'timeout queue 20s\n '
'timeout connect 5s\n '
'timeout http-request 1s\n '
'timeout http-keep-alive 5s\n '
'timeout client 1s\n '
'timeout server 1s\n '
'timeout check 5s\n '
'maxconn 8000\n'
'\nfrontend stats\n '
'mode http\n '
'bind 1.2.3.4:8999\n '
'bind localhost:8999\n '
'stats enable\n '
'stats uri /stats\n '
'stats refresh 10s\n '
'stats auth admin:12345\n '
'http-request use-service prometheus-exporter if { path /metrics }\n '
'monitor-uri /health\n'
'\nfrontend frontend\n '
'bind 1.2.3.4:8089\n '
'default_backend backend\n\n'
'backend backend\n '
'option forwardfor\n '
'balance static-rr\n '
'option httpchk HEAD / HTTP/1.0\n '
'server ' + haproxy_generated_conf[1][0] +
' 1::4:80 check weight 100\n'
}
}
assert haproxy_generated_conf[0] == haproxy_expected_conf
def test_grafana_config(self, _run_cephadm, cephadm_module: CephadmOrchestrator):
_run_cephadm.side_effect = async_side_effect(('{}', '', 0))
with with_host(cephadm_module, 'test'):
cephadm_module.set_store('grafana_crt', 'c')
cephadm_module.set_store('grafana_key', 'k')
with with_service(cephadm_module, MonitoringSpec('prometheus')) as _, \
with_service(cephadm_module, GrafanaSpec('grafana')) as _:
files = {
'grafana.ini': dedent("""
# This file is generated by cephadm.
[users]
default_theme = light
[auth.anonymous]
enabled = true
org_name = 'Main Org.'
org_role = 'Viewer'
[server]
domain = 'bootstrap.storage.lab'
protocol = https
cert_file = /etc/grafana/certs/cert_file
cert_key = /etc/grafana/certs/cert_key
http_port = 3000
http_addr =
[security]
disable_initial_admin_creation = true
cookie_secure = true
cookie_samesite = none
allow_embedding = true""").lstrip(), # noqa: W291
'provisioning/datasources/ceph-dashboard.yml': dedent("""
# This file is generated by cephadm.
deleteDatasources:
- name: 'Dashboard1'
orgId: 1
datasources:
- name: 'Dashboard1'
type: 'prometheus'
access: 'proxy'
orgId: 1
url: 'http://[1::4]:9095'
basicAuth: false
isDefault: true
editable: false
""").lstrip(),
'certs/cert_file': dedent("""
# generated by cephadm
c""").lstrip(),
'certs/cert_key': dedent("""
# generated by cephadm
k""").lstrip(),
}
_run_cephadm.assert_called_with(
'test',
'grafana.test',
'deploy',
[
'--name', 'grafana.test',
'--meta-json',
'{"service_name": "grafana", "ports": [3000], "ip": null, "deployed_by": [], "rank": null, "rank_generation": null}',
'--config-json', '-', '--tcp-ports', '3000'],
stdin=json.dumps({"files": files}),
image='')