def __init__(self) -> None:
self._config = ContainersOptionsModel()
if os.path.exists("./gravel/cephadm/cephadm.bin"):
# dev environment
self.cephadm = ["sudo", "./gravel/cephadm/cephadm.bin"]
else:
# deployment environment
self.cephadm = ["sudo", "cephadm"]
async def test_gather_facts_fail_1(mocker: MockerFixture):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return "fail", "", 0
cephadm = Cephadm(ContainersOptionsModel())
mocker.patch.object(cephadm, "call", side_effect=mock_call)
with pytest.raises(CephadmError):
await cephadm.gather_facts()
async def test_gather_facts_fail_2(mocker: MockerFixture,
get_data_contents: Callable[[str, str],
str]):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, "gather_facts_real.json"), "", 1
cephadm = Cephadm(ContainersOptionsModel())
mocker.patch.object(cephadm, "call", side_effect=mock_call)
with pytest.raises(CephadmError):
await cephadm.gather_facts()
async def test_bootstrap(mocker: MockerFixture):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return "foo", "bar", 0
cephadm = Cephadm(ContainersOptionsModel())
mocker.patch.object(cephadm, "call", side_effect=mock_call)
out, err, rc = await cephadm.bootstrap("127.0.0.1")
assert out == "foo"
assert err == "bar"
assert rc == 0
async def test_gather_facts_real(mocker: MockerFixture,
get_data_contents: Callable[[str, str], str]):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, "gather_facts_real.json"), "", 0
cephadm = Cephadm(ContainersOptionsModel())
mocker.patch.object(cephadm, "call", side_effect=mock_call)
result: HostFactsModel = await cephadm.gather_facts()
real: Dict[str, Any] = json.loads(
get_data_contents(DATA_DIR, "gather_facts_real.json"))
assert result.dict() == real
async def _prepare_node(self, hostname: str,
network: Optional[NetworkConfigModel]) -> None:
"""Prepare the node."""
assert hostname is not None and len(hostname) > 0
assert self._gstate is not None
if network:
logger.debug("Apply network configuration.")
await self._gstate.network.apply_config(network.interfaces,
network.nameservers,
network.routes)
try:
await set_hostname(hostname)
except HostnameCtlError as e:
msg = f"Error setting hostname during join: {e.message}"
logger.error(msg)
raise JoinError(msg)
except Exception as e:
msg = f"Unknown error setting hostname during join: {str(e)}"
logger.error(msg)
raise JoinError(msg)
store = self._gstate.store
await store.ensure_connection()
logger.debug("Obtain NTP address from store.")
ntpaddr = await store.get("/nodes/ntp_addr")
logger.debug(f"NTP address is '{ntpaddr}'")
assert ntpaddr is not None and len(ntpaddr) > 0
try:
await set_ntp_addr(ntpaddr)
except NodeChronyRestartError as e:
msg = f"Error setting NTP address: {e.message}"
logger.error(msg)
raise JoinError(msg)
ctrcfg_str = await store.get("/nodes/containers")
assert ctrcfg_str is not None and len(ctrcfg_str) > 0
ctrcfg = ContainersOptionsModel.parse_raw(ctrcfg_str)
self._gstate.config.options.containers = ctrcfg
try:
await set_registry(ctrcfg.registry, ctrcfg.secure)
except ContainerError as e:
msg = f"Error setting container registry: {e.message}"
logger.error(msg)
raise JoinError(msg)
async def test_volume_inventory(mocker: MockerFixture,
get_data_contents: Callable[[str, str], str]):
async def mock_call(cmd: List[str],
noimage: bool = False,
outcb: Optional[Any] = None) -> Tuple[str, str, int]:
return get_data_contents(DATA_DIR, "inventory_real.json"), "", 0
cephadm = Cephadm(ContainersOptionsModel())
mocker.patch.object(cephadm, "call", side_effect=mock_call)
result: List[VolumeDeviceModel] = await cephadm.get_volume_inventory()
for dev in result:
if dev.sys_api.rotational:
assert dev.human_readable_type == "hdd"
else:
assert dev.human_readable_type == "sdd"
def __init__(self):
super().__init__(ContainersOptionsModel())
class Cephadm:
_config: ContainersOptionsModel
cephadm: List[str]
def __init__(self) -> None:
self._config = ContainersOptionsModel()
if os.path.exists("./gravel/cephadm/cephadm.bin"):
# dev environment
self.cephadm = ["sudo", "./gravel/cephadm/cephadm.bin"]
else:
# deployment environment
self.cephadm = ["sudo", "cephadm"]
def set_config(self, config: ContainersOptionsModel) -> None:
self._config = config
async def call(
self,
cmd: List[str],
noimage: bool = False,
outcb: Optional[Callable[[str], None]] = None,
) -> Tuple[str, str, int]:
assert len(cmd) > 0
cmdlst: List[str] = list(self.cephadm)
if not noimage:
image = self._config.get_image()
cmdlst.extend(["--image", image])
cmdlst.extend(cmd)
logger.debug(f"call with {self.cephadm}")
logger.debug(f"call: {cmdlst}")
process = await asyncio.create_subprocess_exec(
*cmdlst,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
assert process.stdout
assert process.stderr
stdout, stderr = await asyncio.gather(self._tee(process.stdout, outcb),
self._tee(process.stderr, outcb))
retcode = await asyncio.wait_for(process.wait(), None)
return stdout, stderr, retcode
async def run_in_background(self, cmd: List[str]) -> None:
pass
async def _tee(
self,
reader: asyncio.StreamReader,
cb: Optional[Callable[[str], None]] = None,
) -> str:
collected = StringIO()
async for line in reader:
msg = line.decode("utf-8")
collected.write(msg)
if cb:
cb(msg)
return collected.getvalue()
#
# command wrappers
#
async def bootstrap(
self,
addr: str,
percentcb: Optional[Callable[[int],
None]] = None) -> Tuple[str, str, int]:
if not addr:
raise CephadmError("address not specified")
msg_to_percent: List[Tuple[str, int]] = [
("cluster fsid", 5),
("pulling container image", 7),
("extracting ceph user uid/gid from container image", 15),
("creating mon", 20),
("waiting for mon to start", 25),
("wrote config", 40),
("creating mgr", 45),
("waiting for mgr to start", 50),
("enabling cephadm module", 60),
("setting orchestrator backend to cephadm", 70),
("adding host", 80),
("bootstrap complete", 100),
]
def outcb_handler(msg: str) -> None:
if not percentcb:
return
t = msg.lower()
for m, p in msg_to_percent:
if m in t:
percentcb(p)
def get_default_ceph_conf() -> str:
s = ("[global]", "osd_pool_default_size = 2", "")
return "\n".join(s)
def write_tmp(s: str) -> IO[str]:
t = tempfile.NamedTemporaryFile(
prefix="aquarium-",
mode="w",
)
t.write(s)
t.flush()
return t
tmp_config: IO[str] = write_tmp(get_default_ceph_conf())
cmd: List[str] = [
"bootstrap",
"--skip-pull",
"--skip-prepare-host",
"--mon-ip",
addr,
"--config",
tmp_config.name,
"--initial-dashboard-user",
"admin",
"--initial-dashboard-password",
"aquarium",
"--dashboard-password-noupdate",
"--skip-monitoring-stack",
]
return await self.call(cmd, noimage=False, outcb=outcb_handler)
async def gather_facts(self) -> HostFactsModel:
stdout, stderr, rc = await self.call(["gather-facts"], noimage=True)
if rc != 0:
raise CephadmError(stderr)
try:
return HostFactsModel.parse_raw(stdout)
except pydantic.error_wrappers.ValidationError:
raise CephadmError("format error while obtaining facts")
async def get_volume_inventory(self) -> List[VolumeDeviceModel]:
cmd = ["ceph-volume", "inventory", "--format", "json"]
stdout, stderr, rc = await self.call(cmd)
if rc != 0:
raise CephadmError(stderr)
try:
devs = json.loads(stdout)
logger.debug(json.dumps(devs, indent=2))
except json.decoder.JSONDecodeError as e:
raise CephadmError("format error while obtaining inventory") from e
inventory = parse_obj_as(List[VolumeDeviceModel], devs)
for d in inventory:
if not d.human_readable_type:
if d.sys_api.rotational:
d.human_readable_type = "hdd"
else:
d.human_readable_type = "ssd"
return inventory
async def pull_images(self) -> None:
logger.debug("fetching ceph container image")
time_begin: int = int(time.monotonic())
_, stderr, rc = await self.call(["pull"])
if rc != 0:
raise CephadmError(stderr)
time_end: int = int(time.monotonic())
time_diff: int = time_end - time_begin
logger.debug(f"pulled ceph container images: took {time_diff} sec")