async def test_get_node_info(mocker, get_data_contents):
async def mock_facts_call(cmd: str) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, 'gather_facts_real.json'), "", 0
async def mock_inventory_call(cmd: str) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, 'inventory_real.json'), "", 0
cephadm_facts = Cephadm()
mocker.patch.object(cephadm_facts, 'call', side_effect=mock_facts_call)
cephadm_inventory = Cephadm()
mocker.patch.object(cephadm_inventory,
'call',
side_effect=mock_inventory_call)
facts_result = await cephadm_facts.gather_facts()
inventory_result = await cephadm_inventory.get_volume_inventory()
async def mock_facts_result() -> HostFactsModel:
return facts_result
async def mock_inventory_result() -> List[VolumeDeviceModel]:
return inventory_result
cephadm = Cephadm()
mocker.patch.object(cephadm, 'gather_facts', side_effect=mock_facts_result)
mocker.patch.object(cephadm,
'get_volume_inventory',
side_effect=mock_inventory_result)
info: NodeInfoModel = await cephadm.get_node_info()
assert info.hostname == facts_result.hostname
assert info.disks == inventory_result
async def probe(self) -> None:
cephadm: Cephadm = Cephadm()
start: int = int(time.monotonic())
nodeinfo = await cephadm.get_node_info()
diff: int = int(time.monotonic()) - start
logger.info(f"=> inventory probing took {diff} seconds")
self._latest = nodeinfo
async def get_node_info(cephadm: Cephadm) -> NodeInfoModel:
try:
facts, disks = await asyncio.gather(cephadm.gather_facts(),
get_storage_devices())
except CephadmError as e:
raise CephadmError("error obtaining node info") from e
return NodeInfoModel(
hostname=facts.hostname,
model=facts.model,
vendor=facts.vendor,
kernel=facts.kernel,
operating_system=facts.operating_system,
system_uptime=facts.system_uptime,
current_time=facts.timestamp,
cpu=NodeCPUInfoModel(
arch=facts.arch,
model=facts.cpu_model,
cores=facts.cpu_cores,
count=facts.cpu_count,
threads=facts.cpu_threads,
load=NodeCPULoadModel(
one_min=facts.cpu_load["1min"],
five_min=facts.cpu_load["5min"],
fifteen_min=facts.cpu_load["15min"],
),
),
nics=facts.interfaces,
memory=NodeMemoryInfoModel(
available_kb=facts.memory_available_kb,
free_kb=facts.memory_free_kb,
total_kb=facts.memory_total_kb,
),
disks=disks,
)
async def _find_candidate_addr(self) -> str:
logger.debug("bootstrap > find candidate address")
try:
cephadm: Cephadm = Cephadm()
facts = await cephadm.gather_facts()
except Exception as e:
raise NetworkAddressNotFoundError(e)
if len(facts.interfaces) == 0:
logger.error("bootstrap > unable to find interface facts!")
raise NetworkAddressNotFoundError("interfaces not available")
candidates: List[str] = []
for nic in facts.interfaces.values():
if nic.iftype == "loopback":
continue
candidates.append(nic.ipv4_address)
selected: Optional[str] = None
if len(candidates) > 0:
selected = candidates[0]
if selected is None or len(selected) == 0:
raise NetworkAddressNotFoundError("no address available")
netmask_idx = selected.find("/")
if netmask_idx > 0:
selected = selected[:netmask_idx]
return selected
async def _do_bootstrap(
self,
address: str,
cb: Callable[[bool, Optional[str]], Awaitable[None]]
) -> None:
logger.info(f"bootstrap address: {address}")
assert address is not None and len(address) > 0
self._stage = BootstrapStage.RUNNING
def progress_cb(percent: int) -> None:
logger.debug(f"bootstrap > {percent}%")
self._progress = percent
retcode: int = 0
try:
cephadm: Cephadm = Cephadm()
_, _, retcode = await cephadm.bootstrap(address, progress_cb)
except Exception as e:
await cb(False, f"error bootstrapping: {str(e)}")
self._stage = BootstrapStage.ERROR
self._error_code = BootstrapErrorEnum.CANT_BOOTSTRAP
self._error_msg = "error bootstrapping"
return
if retcode != 0:
await cb(False, f"error bootstrapping: rc = {retcode}")
self._stage = BootstrapStage.ERROR
self._error_code = BootstrapErrorEnum.CANT_BOOTSTRAP
self._error_msg = "error bootstrapping"
return
self._stage = BootstrapStage.DONE
await cb(True, None)
async def test_gather_facts_fail_1(mocker):
async def mock_call(cmd: str) -> Tuple[str, str, int]:
return "fail", "", 0
cephadm = Cephadm()
mocker.patch.object(cephadm, 'call', side_effect=mock_call)
with pytest.raises(CephadmError):
await cephadm.gather_facts()
async def test_gather_facts_fail_2(mocker, get_data_contents):
async def mock_call(cmd: str) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, 'gather_facts_real.json'), "", 1
cephadm = Cephadm()
mocker.patch.object(cephadm, 'call', side_effect=mock_call)
with pytest.raises(CephadmError):
await cephadm.gather_facts()
async def test_volume_inventory_fail(mocker):
async def mock_call(cmd: str) -> Tuple[str, str, int]:
return "fail", "", 0
cephadm = Cephadm()
mocker.patch.object(cephadm, 'call', side_effect=mock_call)
with pytest.raises(CephadmError):
await cephadm.get_volume_inventory()
async def test_bootstrap(mocker):
async def mock_call(cmd: str, cb: Optional[Any]) -> Tuple[str, str, int]:
return "foo", "bar", 0
cephadm = Cephadm()
mocker.patch.object(cephadm, 'call', side_effect=mock_call)
out, err, rc = await cephadm.bootstrap("127.0.0.1")
assert out == "foo"
assert err == "bar"
assert rc == 0
async def test_volume_inventory_fail(mocker: MockerFixture):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return "fail", "", 0
cephadm = Cephadm()
mocker.patch.object(cephadm, "call", side_effect=mock_call)
with pytest.raises(CephadmError):
await cephadm.get_volume_inventory()
async def test_gather_facts_fail_2(
mocker: MockerFixture, get_data_contents: Callable[[str, str], str]
):
async def mock_call(cmd: str) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, "gather_facts_real.json"), "", 1
cephadm = Cephadm()
mocker.patch.object(cephadm, "call", side_effect=mock_call)
with pytest.raises(CephadmError):
await cephadm.gather_facts()
async def test_gather_facts_real(mocker, get_data_contents):
async def mock_call(cmd: str) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, 'gather_facts_real.json'), "", 0
cephadm = Cephadm()
mocker.patch.object(cephadm, 'call', side_effect=mock_call)
result: HostFactsModel = await cephadm.gather_facts()
real: Dict[str, Any] = json.loads(
get_data_contents(DATA_DIR, 'gather_facts_real.json'))
assert result.dict() == real
async def test_gather_facts_fail_1(mocker: MockerFixture):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return "fail", "", 0
cephadm = Cephadm(ContainersOptionsModel())
mocker.patch.object(cephadm, "call", side_effect=mock_call)
with pytest.raises(CephadmError):
await cephadm.gather_facts()
async def test_gather_facts_fail_2(mocker: MockerFixture,
get_data_contents: Callable[[str, str],
str]):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, "gather_facts_real.json"), "", 1
cephadm = Cephadm(ContainersOptionsModel())
mocker.patch.object(cephadm, "call", side_effect=mock_call)
with pytest.raises(CephadmError):
await cephadm.gather_facts()
async def test_bootstrap(mocker: MockerFixture):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return "foo", "bar", 0
cephadm = Cephadm(ContainersOptionsModel())
mocker.patch.object(cephadm, "call", side_effect=mock_call)
out, err, rc = await cephadm.bootstrap("127.0.0.1")
assert out == "foo"
assert err == "bar"
assert rc == 0
async def get_node_info() -> NodeInfoModel:
"""
Obtain this node's information and facts.
Lists the node's volumes (same as `/local/volumes`), and additional host
information, such as OS metadata, NICs, etc.
This information is obtained via `cephadm`.
This is a sync call to `cephadm` and may take a while to return.
"""
cephadm = Cephadm()
return await cephadm.get_node_info()
async def test_gather_facts_real(mocker: MockerFixture,
get_data_contents: Callable[[str, str], str]):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, "gather_facts_real.json"), "", 0
cephadm = Cephadm(ContainersOptionsModel())
mocker.patch.object(cephadm, "call", side_effect=mock_call)
result: HostFactsModel = await cephadm.gather_facts()
real: Dict[str, Any] = json.loads(
get_data_contents(DATA_DIR, "gather_facts_real.json"))
assert result.dict() == real
async def aquarium_startup(_: FastAPI, aquarium_api: FastAPI):
lvl = "INFO" if not os.getenv("AQUARIUM_DEBUG") else "DEBUG"
setup_logging(lvl)
logger.info("Aquarium startup!")
gstate: GlobalState = GlobalState()
# init node mgr
logger.info("starting node manager")
nodemgr: NodeMgr = NodeMgr(gstate)
# Prep cephadm
cephadm: Cephadm = Cephadm(gstate.config.options.containers)
gstate.add_cephadm(cephadm)
# Set up Ceph connections
ceph: Ceph = Ceph()
ceph_mgr: Mgr = Mgr(ceph)
gstate.add_ceph_mgr(ceph_mgr)
ceph_mon: Mon = Mon(ceph)
gstate.add_ceph_mon(ceph_mon)
# Set up all of the tickers
devices: Devices = Devices(
gstate.config.options.devices.probe_interval,
nodemgr,
ceph_mgr,
ceph_mon,
)
gstate.add_devices(devices)
status: Status = Status(gstate.config.options.status.probe_interval,
gstate, nodemgr)
gstate.add_status(status)
inventory: Inventory = Inventory(
gstate.config.options.inventory.probe_interval, nodemgr, gstate)
gstate.add_inventory(inventory)
storage: Storage = Storage(gstate.config.options.storage.probe_interval,
nodemgr, ceph_mon)
gstate.add_storage(storage)
await nodemgr.start()
await gstate.start()
# Add instances into FastAPI's state:
aquarium_api.state.gstate = gstate
aquarium_api.state.nodemgr = nodemgr
async def test_volume_inventory(mocker, get_data_contents):
async def mock_call(cmd: str) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, 'inventory_real.json'), "", 0
cephadm = Cephadm()
mocker.patch.object(cephadm, 'call', side_effect=mock_call)
result: List[VolumeDeviceModel] = \
await cephadm.get_volume_inventory()
for dev in result:
if dev.sys_api.rotational:
assert dev.human_readable_type == "hdd"
else:
assert dev.human_readable_type == "sdd"
async def test_volume_inventory(mocker: MockerFixture,
get_data_contents: Callable[[str, str], str]):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, "inventory_real.json"), "", 0
cephadm = Cephadm(ContainersOptionsModel())
mocker.patch.object(cephadm, "call", side_effect=mock_call)
result: List[VolumeDeviceModel] = await cephadm.get_volume_inventory()
for dev in result:
if dev.sys_api.rotational:
assert dev.human_readable_type == "hdd"
else:
assert dev.human_readable_type == "sdd"
async def _do_bootstrap(self, selected_addr: str) -> None:
logger.debug("bootstrap > run in background")
assert selected_addr is not None and len(selected_addr) > 0
self.stage = BootstrapStage.RUNNING
gstate.config.set_deployment_stage(DeploymentStage.bootstrapping)
retcode: int = 0
try:
cephadm: Cephadm = Cephadm()
_, _, retcode = await cephadm.bootstrap(selected_addr)
except Exception as e:
raise BootstrapError(e) from e
if retcode != 0:
raise BootstrapError(f"error bootstrapping: rc = {retcode}")
self.stage = BootstrapStage.DONE
gstate.config.set_deployment_stage(DeploymentStage.bootstrapped)
def gstate_preinit(gstate: GlobalState) -> None:
"""Things that do not require persistent state to work."""
# Prep cephadm
cephadm: Cephadm = Cephadm()
gstate.add_cephadm(cephadm)
gstate.preinit()
async def get_node_info() -> NodeInfoModel:
cephadm = Cephadm()
return await cephadm.get_node_info()
async def get_facts() -> HostFactsModel:
cephadm = Cephadm()
return await cephadm.gather_facts()
async def get_volumes() -> List[VolumeDeviceModel]:
cephadm = Cephadm()
return await cephadm.get_volume_inventory()