async def aquarium_startup(_: FastAPI, aquarium_api: FastAPI):
lvl = "INFO" if not os.getenv("AQUARIUM_DEBUG") else "DEBUG"
setup_logging(lvl)
logger.info("Aquarium startup!")
deployment = DeploymentMgr()
try:
await deployment.preinit()
except DeploymentError as e:
logger.error(f"Unable to pre-init the node: {e.message}")
sys.exit(1)
config: Config = Config()
kvstore: KV = KV()
gstate: GlobalState = GlobalState(config, kvstore)
nodemgr: NodeMgr = NodeMgr(gstate)
gstate_preinit(gstate)
global _main_task
_main_task = asyncio.create_task(
aquarium_main_task(
aquarium_api, config, kvstore, gstate, nodemgr, deployment
)
)
async def test_postbootstrap_config(mocker: MockerFixture,
gstate: GlobalState) -> None:
config_keys: Dict[str, Tuple[str, str]] = {}
def config_set(cls: Any, who: str, name: str, value: str) -> bool:
config_keys[name] = (who, value)
return True
def expect_key(who: str, name: str, value: str) -> None:
assert name in config_keys
scope, val = config_keys[name]
assert scope == who
assert val == value
from gravel.controllers.nodes.mgr import NodeMgr
from gravel.controllers.orch.ceph import Mon
mocker.patch.object(NodeMgr, "_init_state")
mocker.patch.object(Mon, "config_set", new=config_set)
mocker.patch.object(Mon, "call")
mocker.patch.object(Mon, "set_default_ruleset") # ignore default ruleset
mgr = NodeMgr(gstate)
await mgr._post_bootstrap_config()
expect_key("global", "mon_allow_pool_size_one", "true")
expect_key("global", "mon_warn_on_pool_no_redundancy", "false")
expect_key("mgr", "mgr/cephadm/manage_etc_ceph_ceph_conf", "true")
expect_key("global", "auth_allow_insecure_global_id_reclaim", "false")
def test_init_state_fail(gstate: GlobalState,
fs: fake_filesystem.FakeFilesystem) -> None:
from gravel.controllers.nodes.mgr import NodeError
if fs.exists("/etc/aquarium/node.json"):
fs.remove("/etc/aquarium/node.json")
nodemgr = NodeMgr(gstate)
assert fs.exists("/etc/aquarium/node.json")
for f in fs.listdir("/etc/aquarium"):
fs.remove(f"/etc/aquarium/{f}")
assert fs.exists("/etc/aquarium")
fs.rmdir("/etc/aquarium")
fs.create_dir("/etc/aquarium", perm_bits=0o500)
throws = False
try:
nodemgr._init_state()
except NodeError:
throws = True
assert throws
# clean up
for f in fs.listdir("/etc/aquarium"):
fs.remove(f"/etc/aquarium/{f}")
fs.rmdir("/etc/aquarium")
def test_ctor(gstate: GlobalState, fs: fake_filesystem.FakeFilesystem) -> None:
NodeMgr(gstate)
assert fs.exists("/etc/aquarium/node.json")
# clean up
for f in fs.listdir("/etc/aquarium"):
fs.remove(f"/etc/aquarium/{f}")
def test_generate_token(gstate: GlobalState) -> None:
nodemgr = NodeMgr(gstate)
token = nodemgr._generate_token()
assert len(token) > 0
res = token.split("-")
assert len(res) == 4
for s in res:
assert len(s) == 4
def nodemgr(gstate: GlobalState) -> NodeMgr:
from gravel.controllers.nodes.mgr import NodeStateModel
nodemgr = NodeMgr(gstate)
nodemgr._state = NodeStateModel(
uuid="bba35d93-d4a5-48b3-804b-99c406555c89",
address="1.2.3.4",
hostname="foobar",
)
yield nodemgr
async def aquarium_startup(_: FastAPI, aquarium_api: FastAPI):
lvl = "INFO" if not os.getenv("AQUARIUM_DEBUG") else "DEBUG"
setup_logging(lvl)
logger.info("Aquarium startup!")
gstate: GlobalState = GlobalState()
# init node mgr
logger.info("starting node manager")
nodemgr: NodeMgr = NodeMgr(gstate)
# Prep cephadm
cephadm: Cephadm = Cephadm(gstate.config.options.containers)
gstate.add_cephadm(cephadm)
# Set up Ceph connections
ceph: Ceph = Ceph()
ceph_mgr: Mgr = Mgr(ceph)
gstate.add_ceph_mgr(ceph_mgr)
ceph_mon: Mon = Mon(ceph)
gstate.add_ceph_mon(ceph_mon)
# Set up all of the tickers
devices: Devices = Devices(
gstate.config.options.devices.probe_interval,
nodemgr,
ceph_mgr,
ceph_mon,
)
gstate.add_devices(devices)
status: Status = Status(gstate.config.options.status.probe_interval,
gstate, nodemgr)
gstate.add_status(status)
inventory: Inventory = Inventory(
gstate.config.options.inventory.probe_interval, nodemgr, gstate)
gstate.add_inventory(inventory)
storage: Storage = Storage(gstate.config.options.storage.probe_interval,
nodemgr, ceph_mon)
gstate.add_storage(storage)
await nodemgr.start()
await gstate.start()
# Add instances into FastAPI's state:
aquarium_api.state.gstate = gstate
aquarium_api.state.nodemgr = nodemgr
def test_node_shutdown(gstate: GlobalState, mocker: MockerFixture) -> None:
from gravel.controllers.nodes.mgr import NodeInitStage, NodeMgr
class FakeTask:
called = False
def cancel(self) -> None:
self.called = True
nodemgr = NodeMgr(gstate)
nodemgr._incoming_task = FakeTask() # type: ignore
nodemgr._init_stage = NodeInitStage.NONE
nodemgr._node_shutdown()
assert nodemgr._init_stage == NodeInitStage.STOPPING
assert nodemgr._incoming_task.called
def test_fail_ctor(gstate: GlobalState,
fs: fake_filesystem.FakeFilesystem) -> None:
from gravel.controllers.nodes.mgr import NodeError
if fs.exists("/etc/aquarium/node.json"):
fs.remove("/etc/aquarium/node.json")
fs.create_dir("/etc/aquarium/node.json")
throws = False
try:
NodeMgr(gstate)
except NodeError:
throws = True
except Exception:
assert False
assert throws
# clean up
fs.rmdir("/etc/aquarium/node.json")
async def test_mgr_start(
gstate: GlobalState,
fs: fake_filesystem.FakeFilesystem,
mocker: MockerFixture,
) -> None:
from gravel.controllers.nodes.deployment import NodeStageEnum
from gravel.controllers.nodes.mgr import NodeError, NodeStateModel
nodemgr = NodeMgr(gstate)
assert nodemgr._state
assert nodemgr.deployment_state.can_start()
orig = nodemgr.deployment_state.can_start
nodemgr.deployment_state.can_start = mocker.MagicMock(return_value=False)
throws = False
try:
await nodemgr.start()
except NodeError as e:
assert "unstartable" in e.message
throws = True
assert throws
nodemgr.deployment_state.can_start = orig
nodemgr._deployment._state._stage = NodeStageEnum.NONE
nodemgr._node_prepare = mocker.AsyncMock()
await nodemgr.start()
nodemgr._node_prepare.assert_called_once() # type: ignore
nodemgr._deployment._state._stage = NodeStageEnum.READY
nodemgr._state = NodeStateModel(
uuid="bba35d93-d4a5-48b3-804b-99c406555c89",
address="1.2.3.4",
hostname="foobar",
)
nodemgr._start_ceph = mocker.AsyncMock()
nodemgr._node_start = mocker.AsyncMock()
await nodemgr.start()
nodemgr._start_ceph.assert_called_once() # type: ignore
nodemgr._node_start.assert_called_once() # type: ignore
async def test_join_checks(gstate: GlobalState) -> None:
from gravel.controllers.nodes.mgr import (
JoinParamsModel,
NodeCantJoinError,
NodeError,
NodeInitStage,
NodeNotStartedError,
)
nodemgr = NodeMgr(gstate)
throws = False
nodemgr._init_stage = NodeInitStage.NONE
try:
await nodemgr.join(
"1.2.3.4", "751b-51fd-10d7-f7b4", JoinParamsModel(hostname="foobar")
)
except NodeNotStartedError:
throws = True
assert throws
throws = False
nodemgr._init_stage = NodeInitStage.STARTED
try:
await nodemgr.join(
"1.2.3.4", "751b-51fd-10d7-f7b4", JoinParamsModel(hostname="foobar")
)
except NodeCantJoinError:
throws = True
assert throws
throws = False
nodemgr._init_stage = NodeInitStage.AVAILABLE
try:
await nodemgr.join(
"1.2.3.4", "751b-51fd-10d7-f7b4", JoinParamsModel(hostname="")
)
except NodeError as e:
throws = True
assert "hostname" in e.message
assert throws
async def test_start_ceph(gstate: GlobalState, mocker: MockerFixture) -> None:
from gravel.controllers.nodes.mgr import NodeError
called = False
async def mock_call(
cmd: List[str],
) -> Tuple[int, Optional[str], Optional[str]]:
nonlocal called
called = True
assert cmd[0] == "systemctl"
assert cmd[1] == "start"
assert cmd[2] == "ceph.target"
return 0, None, None
mocker.patch("gravel.controllers.nodes.mgr.aqr_run_cmd", new=mock_call)
nodemgr = NodeMgr(gstate)
await nodemgr._start_ceph()
assert called
called = False
async def fail_call(
cmd: List[str],
) -> Tuple[int, Optional[str], Optional[str]]:
nonlocal called
called = True
return 1, None, "oops"
mocker.patch("gravel.controllers.nodes.mgr.aqr_run_cmd", new=fail_call)
throwed = False
try:
await nodemgr._start_ceph()
except NodeError as e:
assert "oops" in e.message
throwed = True
assert called
assert throwed
async def test_obtain_images(
gstate: GlobalState, mocker: MockerFixture
) -> None:
orig_cephadm_pull_img = gstate.cephadm.pull_images
gstate.cephadm.pull_images = mocker.AsyncMock()
nodemgr = NodeMgr(gstate)
ret = await nodemgr._obtain_images()
assert ret
gstate.cephadm.pull_images.assert_called_once() # type: ignore
from gravel.cephadm.cephadm import CephadmError
gstate.cephadm.pull_images = mocker.AsyncMock(
side_effect=CephadmError("foobar")
)
ret = await nodemgr._obtain_images()
assert not ret
gstate.cephadm.pull_images.assert_called_once() # type: ignore
gstate.cephadm.pull_images = orig_cephadm_pull_img