class CephFS:
mgr: Mgr
mon: Mon
def __init__(self):
self.mgr = Mgr()
self.mon = Mon()
pass
def create(self, name: str) -> None:
cmd = {
"prefix": "fs volume create",
"name": name
}
try:
res = self.mgr.call(cmd)
except CephCommandError as e:
raise CephFSError(e) from e
# this command does not support json at this time, and will output
# free-form text instead. We are not going to parse it, but we'll make
# sure we've got something out of it.
assert "result" in res
assert len(res["result"]) > 0
def volume_ls(self) -> CephFSVolumeListModel:
cmd = {
"prefix": "fs volume ls",
"format": "json"
}
try:
res = self.mgr.call(cmd)
except CephCommandError as e:
raise CephFSError(e) from e
return CephFSVolumeListModel(
volumes=parse_obj_as(List[CephFSNameModel], res)
)
def ls(self) -> List[CephFSListEntryModel]:
cmd = {
"prefix": "fs ls",
"format": "json"
}
try:
res = self.mon.call(cmd)
except CephCommandError as e:
raise CephFSError(e) from e
return parse_obj_as(List[CephFSListEntryModel], res)
def get_fs_info(self, name: str) -> CephFSListEntryModel:
ls: List[CephFSListEntryModel] = self.ls()
for fs in ls:
if fs.name == name:
return fs
raise CephFSError(f"unknown filesystem {name}")
class Orchestrator:
cluster: Mgr
def __init__(self):
self.cluster = Mgr()
def call(self, cmd: Dict[str, Any]) -> Any:
if "format" not in cmd:
cmd["format"] = "json"
return self.cluster.call(cmd)
def host_ls(self) -> List[OrchHostListModel]:
cmd = {"prefix": "orch host ls"}
res = self.call(cmd)
return parse_obj_as(List[OrchHostListModel], res)
def devices_ls(self) -> List[OrchDevicesPerHostModel]:
cmd = {"prefix": "orch device ls"}
res = self.call(cmd)
return parse_obj_as(List[OrchDevicesPerHostModel], res)
def assimilate_all_devices(self) -> None:
cmd = {"prefix": "orch apply osd", "all_available_devices": True}
res = self.call(cmd)
assert "result" in res
def all_devices_assimilated(self) -> bool:
res = self.devices_ls()
for host in res:
for dev in host.devices:
if dev.available:
return False
return True
def apply_mds(self, fsname: str) -> None:
cmd = {"prefix": "orch apply mds", "fs_name": fsname}
self.call(cmd)
def get_public_key(self) -> str:
cmd = {"prefix": "cephadm get-pub-key"}
res = self.call(cmd)
assert "result" in res
return res["result"]
def host_add(self, hostname: str, address: str) -> bool:
assert hostname
assert address
cmd = {
"prefix": "orch host add",
"hostname": hostname,
"addr": address
}
try:
self.call(cmd)
except CephCommandError:
logger.error(f"host add > unable to add {hostname} {address}")
return False
return True
class NFSService:
mgr: Mgr
def __init__(self):
self.mgr = Mgr()
def _call(self, cmd: Dict[str, Any]) -> Any:
try:
return self.mgr.call(cmd)
except CephCommandError as e:
raise NFSError(e) from e
def create(self, name: str, placement: Optional[str]) -> str:
cmd = {
'prefix': 'nfs cluster create',
'type':
'cephfs', # TODO: pending https://github.com/ceph/ceph/pull/40411
'clusterid': name,
}
if placement:
cmd['placement'] = placement
return self._call(cmd)['result']
def update(self, name: str, placement: str) -> str:
res = self._call({
'prefix': 'nfs cluster update',
'clusterid': name,
'placement': placement,
})
return res['result']
def delete(self, name: str) -> str:
res = self._call({
'prefix': 'nfs cluster delete',
'clusterid': name,
})
return res['result']
def ls(self) -> List[str]:
res = self._call({
'prefix': 'nfs cluster ls',
'format': 'json',
})
return res['result'].split() if res.get('result') else []
def info(self, name: Optional[str] = None) -> List[NFSServiceModel]:
cmd = {
'prefix': 'nfs cluster info',
'format': 'json',
}
if name:
cmd['clusterid'] = name
res = self._call(cmd)
ret: List[NFSServiceModel] = []
for name in res:
daemons = parse_obj_as(List[NFSDaemonModel], res[name])
ret.append(NFSServiceModel(name=name, daemons=daemons))
return ret
class CephFS:
mgr: Mgr
mon: Mon
def __init__(self):
self.mgr = Mgr()
self.mon = Mon()
pass
def create(self, name: str) -> None:
cmd = {"prefix": "fs volume create", "name": name}
try:
# this is expected to be a silent command
self.mgr.call(cmd)
except CephCommandError as e:
raise CephFSError(e) from e
# schedule orchestrator to update the number of mds instances
orch = Orchestrator()
orch.apply_mds(name)
def volume_ls(self) -> CephFSVolumeListModel:
cmd = {"prefix": "fs volume ls", "format": "json"}
try:
res = self.mgr.call(cmd)
except CephCommandError as e:
raise CephFSError(e) from e
return CephFSVolumeListModel(
volumes=parse_obj_as(List[CephFSNameModel], res))
def ls(self) -> List[CephFSListEntryModel]:
cmd = {"prefix": "fs ls", "format": "json"}
try:
res = self.mon.call(cmd)
except CephCommandError as e:
raise CephFSError(e) from e
return parse_obj_as(List[CephFSListEntryModel], res)
def get_fs_info(self, name: str) -> CephFSListEntryModel:
ls: List[CephFSListEntryModel] = self.ls()
for fs in ls:
if fs.name == name:
return fs
raise CephFSError(f"unknown filesystem {name}")
def test_ceph_conf(fs: fake_filesystem.FakeFilesystem):
# default location
fs.add_real_file( # pyright: reportUnknownMemberType=false
os.path.join(TEST_DIR, 'data/default_ceph.conf'),
target_path='/etc/ceph/ceph.conf'
)
Mgr()
Mon()
# custom location
conf_file = '/foo/bar/baz.conf'
fs.add_real_file(
os.path.join(TEST_DIR, 'data/default_ceph.conf'),
target_path=conf_file
)
Mgr(conf_file=conf_file)
Mon(conf_file=conf_file)
# invalid location
conf_file = "missing.conf"
with pytest.raises(FileNotFoundError, match=conf_file):
Mgr(conf_file=conf_file)
Mon(conf_file=conf_file)
async def aquarium_startup(_: FastAPI, aquarium_api: FastAPI):
lvl = "INFO" if not os.getenv("AQUARIUM_DEBUG") else "DEBUG"
setup_logging(lvl)
logger.info("Aquarium startup!")
gstate: GlobalState = GlobalState()
# init node mgr
logger.info("starting node manager")
nodemgr: NodeMgr = NodeMgr(gstate)
# Prep cephadm
cephadm: Cephadm = Cephadm(gstate.config.options.containers)
gstate.add_cephadm(cephadm)
# Set up Ceph connections
ceph: Ceph = Ceph()
ceph_mgr: Mgr = Mgr(ceph)
gstate.add_ceph_mgr(ceph_mgr)
ceph_mon: Mon = Mon(ceph)
gstate.add_ceph_mon(ceph_mon)
# Set up all of the tickers
devices: Devices = Devices(
gstate.config.options.devices.probe_interval,
nodemgr,
ceph_mgr,
ceph_mon,
)
gstate.add_devices(devices)
status: Status = Status(gstate.config.options.status.probe_interval,
gstate, nodemgr)
gstate.add_status(status)
inventory: Inventory = Inventory(
gstate.config.options.inventory.probe_interval, nodemgr, gstate)
gstate.add_inventory(inventory)
storage: Storage = Storage(gstate.config.options.storage.probe_interval,
nodemgr, ceph_mon)
gstate.add_storage(storage)
await nodemgr.start()
await gstate.start()
# Add instances into FastAPI's state:
aquarium_api.state.gstate = gstate
aquarium_api.state.nodemgr = nodemgr
class Orchestrator:
cluster: Mgr
def __init__(self):
self.cluster = Mgr()
def call(self, cmd: Dict[str, Any]) -> Any:
if "format" not in cmd:
cmd["format"] = "json"
return self.cluster.call(cmd)
def host_ls(self) -> List[OrchHostListModel]:
cmd = {"prefix": "orch host ls"}
res = self.call(cmd)
return parse_obj_as(List[OrchHostListModel], res)
def devices_ls(self) -> List[OrchDevicesPerHostModel]:
cmd = {"prefix": "orch device ls"}
res = self.call(cmd)
return parse_obj_as(List[OrchDevicesPerHostModel], res)
def assimilate_all_devices(self) -> None:
cmd = {
"prefix": "orch apply osd",
"all_available_devices": True
}
res = self.call(cmd)
assert "result" in res
def all_devices_assimilated(self) -> bool:
res = self.devices_ls()
for host in res:
for dev in host.devices:
if dev.available:
return False
return True
def __init__(self):
self.cluster = Mgr()
def __init__(self):
self.mgr = Mgr()
self.mon = Mon()
pass
async def startup(aquarium_app: FastAPI, aquarium_api: FastAPI):
from fastapi.logger import logger as fastapi_logger
from gravel.cephadm.cephadm import Cephadm
from gravel.controllers.inventory.inventory import Inventory
from gravel.controllers.nodes.deployment import NodeDeployment
from gravel.controllers.nodes.errors import NodeCantDeployError
from gravel.controllers.nodes.mgr import (
NodeError,
NodeInitStage,
NodeMgr,
)
from gravel.controllers.orch.ceph import Ceph, Mgr, Mon
from gravel.controllers.resources.devices import Devices
from gravel.controllers.resources.status import Status
from gravel.controllers.resources.storage import Storage
logger: logging.Logger = fastapi_logger
class FakeNodeDeployment(NodeDeployment):
# Do we still need this thing since removing etcd?
pass
class FakeNodeMgr(NodeMgr):
def __init__(self, gstate: GlobalState):
super().__init__(gstate)
self._deployment = FakeNodeDeployment(gstate, self._connmgr)
async def start(self) -> None:
assert self._state
logger.debug(f"start > {self._state}")
if not self.deployment_state.can_start():
raise NodeError("unable to start unstartable node")
assert self._init_stage == NodeInitStage.NONE
if self.deployment_state.nostage:
await self._node_prepare()
else:
assert (self.deployment_state.ready
or self.deployment_state.deployed)
assert self._state.hostname
assert self._state.address
await self.gstate.store.ensure_connection()
async def _obtain_images(self) -> bool:
return True
class FakeCephadm(Cephadm):
def __init__(self):
super().__init__(ContainersOptionsModel())
async def call(
self,
cmd: List[str],
noimage: bool = False,
outcb: Optional[Callable[[str], None]] = None,
) -> Tuple[str, str, int]:
# Implement expected calls to cephadm with testable responses
if cmd[0] == "pull":
return "", "", 0
elif cmd[0] == "gather-facts":
return (
get_data_contents(DATA_DIR, "gather_facts_real.json"),
"",
0,
)
elif cmd == ["ceph-volume", "inventory", "--format", "json"]:
return (
get_data_contents(DATA_DIR, "inventory_real.json"),
"",
0,
)
else:
print(cmd)
print(outcb)
raise Exception("Tests should not get here")
class FakeCeph(Ceph):
def __init__(self, conf_file: str = "/etc/ceph/ceph.conf"):
self.conf_file = conf_file
self._is_connected = False
def connect(self):
if not self.is_connected():
self.cluster = mocker.Mock()
self._is_connected = True
class FakeStorage(Storage): # type: ignore
available = 2000 # type: ignore
total = 2000 # type: ignore
gstate: GlobalState = GlobalState(FakeKV)
# init node mgr
nodemgr: NodeMgr = FakeNodeMgr(gstate)
# Prep cephadm
cephadm: Cephadm = FakeCephadm()
gstate.add_cephadm(cephadm)
# Set up Ceph connections
ceph: Ceph = FakeCeph()
ceph_mgr: Mgr = Mgr(ceph)
gstate.add_ceph_mgr(ceph_mgr)
ceph_mon: Mon = Mon(ceph)
gstate.add_ceph_mon(ceph_mon)
# Set up all of the tickers
devices: Devices = Devices(
gstate.config.options.devices.probe_interval,
nodemgr,
ceph_mgr,
ceph_mon,
)
gstate.add_devices(devices)
status: Status = Status(gstate.config.options.status.probe_interval,
gstate, nodemgr)
gstate.add_status(status)
inventory: Inventory = Inventory(
gstate.config.options.inventory.probe_interval, nodemgr, gstate)
gstate.add_inventory(inventory)
storage: Storage = FakeStorage(
gstate.config.options.storage.probe_interval, nodemgr, ceph_mon)
gstate.add_storage(storage)
await nodemgr.start()
await gstate.start()
# Add instances into FastAPI's state:
aquarium_api.state.gstate = gstate
aquarium_api.state.nodemgr = nodemgr
def __init__(self):
self.mgr = Mgr()
# project aquarium's backend
# Copyright (C) 2021 SUSE, LLC.
from gravel.controllers.orch.ceph import Ceph, Mgr, Mon
from gravel.controllers.orch.cephfs import CephFS, CephFSError
if __name__ == "__main__":
ceph: Ceph = Ceph()
ceph_mgr: Mgr = Mgr(ceph)
ceph_mon: Mon = Mon(ceph)
cephfs: CephFS = CephFS(ceph_mgr, ceph_mon)
try:
cephfs.create("foobarbaz")
except CephFSError as e:
print(f"error: {str(e)}")
res = cephfs.volume_ls()
print(res.json())
print(cephfs.ls())
def test_ceph_conf(self):
with pytest.raises(FileNotFoundError, match="ceph.conf"):
Mgr()
Mon()
class CephFS:
mgr: Mgr
mon: Mon
def __init__(self):
self.mgr = Mgr()
self.mon = Mon()
pass
def create(self, name: str) -> None:
cmd = {"prefix": "fs volume create", "name": name}
try:
# this is expected to be a silent command
self.mgr.call(cmd)
except CephCommandError as e:
raise CephFSError(e) from e
# schedule orchestrator to update the number of mds instances
orch = Orchestrator()
orch.apply_mds(name)
def volume_ls(self) -> CephFSVolumeListModel:
cmd = {"prefix": "fs volume ls", "format": "json"}
try:
res = self.mgr.call(cmd)
except CephCommandError as e:
raise CephFSError(e) from e
return CephFSVolumeListModel(
volumes=parse_obj_as(List[CephFSNameModel], res))
def ls(self) -> List[CephFSListEntryModel]:
cmd = {"prefix": "fs ls", "format": "json"}
try:
res = self.mon.call(cmd)
except CephCommandError as e:
raise CephFSError(e) from e
return parse_obj_as(List[CephFSListEntryModel], res)
def get_fs_info(self, name: str) -> CephFSListEntryModel:
ls: List[CephFSListEntryModel] = self.ls()
for fs in ls:
if fs.name == name:
return fs
raise CephFSError(f"unknown filesystem {name}")
def authorize(self, fsname: str,
clientid: str) -> CephFSAuthorizationModel:
assert fsname and clientid
cmd = {
"prefix": "fs authorize",
"filesystem": fsname,
"entity": f"client.{fsname}-{clientid}",
"caps": ["/", "rw"],
"format": "json"
}
try:
res = self.mon.call(cmd)
except CephCommandError as e:
raise CephFSError(str(e)) from e
lst = parse_obj_as(List[CephFSAuthorizationModel], res)
assert len(lst) == 1
return lst[0]
def get_authorization(self, fsname: str,
clientid: Optional[str]) -> CephFSAuthorizationModel:
if not clientid:
clientid = "default"
cmd = {
"prefix": "auth get",
"entity": f"client.{fsname}-{clientid}",
"format": "json"
}
try:
res = self.mon.call(cmd)
except CephCommandError as e:
if e.rc == errno.ENOENT:
raise CephFSNoAuthorizationError(e.message)
raise CephFSError(str(e)) from e
lst = parse_obj_as(List[CephFSAuthorizationModel], res)
if len(lst) == 0:
raise CephFSNoAuthorizationError()
return lst[0]