async def get_cluster_view(client: AdminAPI) -> ClusterView:
"""
Returns ClusterView object
"""
(nodes_config_resp, nodes_state_resp,
maintenances_resp) = await asyncio.gather(
admin_api.get_nodes_config(client),
admin_api.get_nodes_state(client),
admin_api.get_maintenances(client),
return_exceptions=True,
)
if isinstance(maintenances_resp, NotSupported):
# This exception can be raised from cluster which does not support
# MaintenanceManager yet
maintenances = []
elif isinstance(maintenances_resp, Exception):
raise maintenances_resp
else:
# pyre-fixme[16]: `BaseException` has no attribute `maintenances`.
maintenances = maintenances_resp.maintenances
if isinstance(nodes_config_resp, Exception):
raise nodes_config_resp
if isinstance(nodes_state_resp, Exception):
raise nodes_state_resp
return ClusterView(
# pyre-fixme[16]: `BaseException` has no attribute `nodes`.
nodes_config=nodes_config_resp.nodes,
# pyre-fixme[16]: `BaseException` has no attribute `states`.
nodes_state=nodes_state_resp.states,
maintenances=maintenances,
)
def _combine(
cv: ClusterView,
shards: Optional[List[str]] = None,
node_names: Optional[List[str]] = None,
node_indexes: Optional[List[int]] = None,
) -> Tuple[ShardID, ...]:
shards = list(shards or [])
node_names = list(node_names or [])
node_indexes = list(node_indexes or [])
shard_ids = parse_shards(shards)
for nn in node_names:
shard_ids.add(ShardID(node=cv.get_node_id(node_name=nn), shard_index=-1))
for ni in node_indexes:
shard_ids.add(ShardID(node=NodeID(node_index=ni), shard_index=-1))
shard_ids_expanded = cv.expand_shards(shard_ids)
return shard_ids_expanded
def one_node(mv: MaintenanceView, cv: ClusterView,
ni: int) -> str:
def shards_table(mv: MaintenanceView, cv: ClusterView,
ni: int) -> str:
headers = [
"SHARD INDEX",
"CURRENT STATE",
"TARGET STATE",
"MAINTENANCE STATUS",
"LAST UPDATED",
]
tbl = []
for shard in mv.get_shards_by_node_index(ni):
target_state = mv.shard_target_state
shard_state = mv.get_shard_state(shard)
cur_op_state = shard_state.current_operational_state
current_state = colored(
cur_op_state.name,
_color_shard_op_state(
cur_op_state,
# pyre-ignore
mv.shard_target_state,
),
)
mnt_status = mv.get_shard_maintenance_status(shard)
maintenance_status = colored(
mnt_status.name, _color(mnt_status))
last_updated_at = mv.get_shard_last_updated_at(
shard)
if last_updated_at:
last_updated = (
f"{last_updated_at} "
f"({naturaltime(last_updated_at)})")
else:
last_updated = "-"
tbl.append([
shard.shard_index,
current_state,
# pyre-ignore
target_state.name,
maintenance_status,
last_updated,
])
return tabulate(tbl, headers=headers, tablefmt="plain")
nv = cv.get_node_view(node_index=ni)
return "N{node_index} ({node_name}):\n{shards_table}".format(
node_index=nv.node_index,
node_name=nv.node_name,
shards_table=indent(shards_table(mv, cv, ni), " "),
)
def sequencers(mv: MaintenanceView, cv: ClusterView) -> Optional[str]:
if not mv.affects_sequencers:
return None
headers = [
"NODE INDEX",
"NODE NAME",
"LOCATION",
"TARGET STATE",
"CURRENT STATE",
"MAINTENANCE STATUS",
"LAST UPDATED",
]
tbl = []
for n in mv.sequencer_nodes:
nv = cv.get_node_view(node_index=n.node_index)
node_index = nv.node_index
node_name = nv.node_name
location = nv.location
# pyre-ignore
target_state = mv.sequencer_target_state.name
mnt_status = mv.get_sequencer_maintenance_status(n)
current_state = colored(
# pyre-ignore
nv.sequencer_state.state.name,
_color(mnt_status),
)
maintenance_status = colored(mnt_status.name,
_color(mnt_status))
last_updated_at = mv.get_sequencer_last_updated_at(n)
if last_updated_at:
last_updated = f"{last_updated_at} ({naturaltime(last_updated_at)})"
else:
last_updated = "-"
tbl.append([
node_index,
node_name,
location,
target_state,
current_state,
maintenance_status,
last_updated,
])
return "Sequencer Maintenances:\n{}".format(
indent(tabulate(tbl, headers=headers, tablefmt="plain"),
prefix=" "))
def aggregated(mv: MaintenanceView, cv: ClusterView) -> str:
headers = [
"NODE INDEX",
"NODE NAME",
"LOCATION",
"TARGET STATE",
"CURRENT STATE",
"MAINTENANCE STATUS",
"LAST UPDATED",
]
tbl = []
for ni in mv.affected_storage_node_indexes:
nv = cv.get_node_view(node_index=ni)
node_index = ni
node_name = nv.node_name
location = nv.location
target_state = (
# pyre-ignore
f"{mv.shard_target_state.name}"
f"({len(mv.get_shards_by_node_index(ni))})")
# current_state
chunks = []
for cur_op_state, num in sorted(
Counter(
(mv.get_shard_state(
shard).current_operational_state
for shard in mv.get_shards_by_node_index(ni)
)).items(),
key=lambda x: x[0].name,
):
chunks.append(
colored(
f"{cur_op_state.name}({num})",
_color_shard_op_state(
cur_op_state,
# pyre-ignore
mv.shard_target_state,
),
))
current_state = ",".join(chunks)
# maintenance status
mnt_statuses = [
mv.get_shard_maintenance_status(shard)
for shard in mv.get_shards_by_node_index(ni)
]
chunks = []
for mnt_status, num in sorted(
Counter(mnt_statuses).items(),
key=lambda x: x[0].name):
chunks.append(
colored(f"{mnt_status.name}({num})",
_color(mnt_status)))
maintenance_status = ",".join(chunks)
last_updated_at_time = min((
ShardMaintenanceProgress.from_thrift(
# pyre-ignore
ss.maintenance).last_updated_at
for ss in nv.shard_states if ss.maintenance))
last_updated_at = (
f"{last_updated_at_time} ({naturaltime(last_updated_at_time)})"
)
tbl.append([
node_index,
node_name,
location,
target_state,
current_state,
maintenance_status,
last_updated_at,
])
return tabulate(tabular_data=tbl,
headers=headers,
tablefmt="plain")
def _validate(
self,
cv: ClusterView,
ncs: List[NodeConfig],
nss: List[NodeState],
mnts: Tuple[MaintenanceDefinition, ...],
):
nis = sorted(nc.node_index for nc in ncs)
ni_to_nc = {nc.node_index: nc for nc in ncs}
ni_to_ns = {ns.node_index: ns for ns in nss}
ni_to_mnts: Dict[int,
List[MaintenanceDefinition]] = {ni: []
for ni in nis}
for mnt in mnts:
mnt_nis = set()
for s in mnt.shards:
assert s.node.node_index is not None
mnt_nis.add(s.node.node_index)
for n in mnt.sequencer_nodes:
assert n.node_index is not None
mnt_nis.add(n.node_index)
for ni in mnt_nis:
ni_to_mnts[ni].append(mnt)
self.assertEqual(sorted(cv.get_all_node_indexes()),
sorted(ni_to_nc.keys()))
self.assertEqual(
sorted(cv.get_all_node_views(),
key=operator.attrgetter("node_index")),
sorted(
(NodeView(
node_config=ni_to_nc[ni],
node_state=ni_to_ns[ni],
maintenances=tuple(ni_to_mnts[ni]),
) for ni in ni_to_nc.keys()),
key=operator.attrgetter("node_index"),
),
)
self.assertEqual(sorted(cv.get_all_node_names()),
sorted(nc.name for nc in ncs))
self.assertEqual(sorted(cv.get_all_maintenance_ids()),
sorted(mnt.group_id for mnt in mnts))
self.assertEqual(
sorted(cv.get_all_maintenances(),
key=operator.attrgetter("group_id")),
sorted(mnts, key=operator.attrgetter("group_id")),
)
for ni in nis:
nn = ni_to_nc[ni].name
nc = ni_to_nc[ni]
ns = ni_to_ns[ni]
mnts = tuple(ni_to_mnts[ni])
nv = NodeView(node_config=ni_to_nc[ni],
node_state=ni_to_ns[ni],
maintenances=mnts)
self.assertEqual(cv.get_node_view_by_node_index(ni), nv)
self.assertEqual(cv.get_node_name_by_node_index(ni), nn)
self.assertEqual(cv.get_node_config_by_node_index(ni), nc)
self.assertEqual(cv.get_node_state_by_node_index(ni), ns)
self.assertEqual(cv.get_maintenances_by_node_index(ni), mnts)
self.assertEqual(cv.get_node_view_by_node_name(nn), nv)
self.assertEqual(cv.get_node_index_by_node_name(nn), ni)
self.assertEqual(cv.get_node_config_by_node_name(nn), nc)
self.assertEqual(cv.get_node_state_by_node_name(nn), ns)
self.assertEqual(cv.get_maintenances_by_node_name(nn), mnts)
self.assertEqual(cv.get_node_view(node_name=nn), nv)
self.assertEqual(cv.get_node_index(node_name=nn), ni)
self.assertEqual(cv.get_node_config(node_name=nn), nc)
self.assertEqual(cv.get_node_state(node_name=nn), ns)
self.assertEqual(cv.get_maintenances(node_name=nn), mnts)
self.assertEqual(cv.get_node_view(node_index=ni), nv)
self.assertEqual(cv.get_node_name(node_index=ni), nn)
self.assertEqual(cv.get_node_config(node_index=ni), nc)
self.assertEqual(cv.get_node_state(node_index=ni), ns)
self.assertEqual(cv.get_maintenances(node_index=ni), mnts)
with self.assertRaises(ValueError):
cv.get_node_view(None, None)
with self.assertRaises(ValueError):
cv.get_node_config(None, None)
with self.assertRaises(ValueError):
cv.get_node_state(None, None)
with self.assertRaises(ValueError):
cv.get_maintenances(None, None)
# mismatch node_index and node_name
if len(nis) > 1:
nn = ni_to_nc[nis[0]].name
ni = nis[1]
with self.assertRaises(ValueError):
cv.get_node_view(ni, nn)
with self.assertRaises(ValueError):
cv.get_node_config(ni, nn)
with self.assertRaises(ValueError):
cv.get_node_state(ni, nn)
with self.assertRaises(ValueError):
cv.get_maintenances(ni, nn)
# non-existent node_index
with self.assertRaises(KeyError):
cv.get_node_view(node_index=max(nis) + 1)
# non-existent node_name
with self.assertRaises(KeyError):
nns = {nc.name for nc in ncs}
while True:
nn = gen_word()
if nn not in nns:
break
cv.get_node_view(node_name=nn)
for mnt in mnts:
assert mnt.group_id is not None
self.assertEqual(cv.get_maintenance_by_id(mnt.group_id), mnt)
def _validate(
self,
cv: ClusterView,
ncs: List[NodeConfig],
nss: List[NodeState],
mnts: Tuple[MaintenanceDefinition, ...],
) -> None:
nis = sorted(nc.node_index for nc in ncs)
ni_to_nc = {nc.node_index: nc for nc in ncs}
ni_to_ns = {ns.node_index: ns for ns in nss}
ni_to_mnts: Dict[int,
List[MaintenanceDefinition]] = {ni: []
for ni in nis}
for mnt in mnts:
mnt_nis = set()
for s in mnt.shards:
assert s.node.node_index is not None
mnt_nis.add(s.node.node_index)
for n in mnt.sequencer_nodes:
assert n.node_index is not None
mnt_nis.add(n.node_index)
for ni in mnt_nis:
ni_to_mnts[ni].append(mnt)
self.assertEqual(sorted(cv.get_all_node_indexes()),
sorted(ni_to_nc.keys()))
self.assertEqual(
sorted(cv.get_all_node_views(),
key=operator.attrgetter("node_index")),
sorted(
(NodeView(
node_config=ni_to_nc[ni],
node_state=ni_to_ns[ni],
maintenances=tuple(ni_to_mnts[ni]),
) for ni in ni_to_nc.keys()),
key=operator.attrgetter("node_index"),
),
)
self.assertEqual(sorted(cv.get_all_node_names()),
sorted(nc.name for nc in ncs))
self.assertEqual(
sorted(cv.get_all_maintenance_ids()),
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Generator[typing.Optional[str], None, None]`.
sorted(mnt.group_id for mnt in mnts),
)
self.assertEqual(
sorted(cv.get_all_maintenances(),
key=operator.attrgetter("group_id")),
sorted(mnts, key=operator.attrgetter("group_id")),
)
for ni in nis:
nn = ni_to_nc[ni].name
nc = ni_to_nc[ni]
ns = ni_to_ns[ni]
node_mnts = tuple(ni_to_mnts[ni])
nv = NodeView(
node_config=ni_to_nc[ni],
node_state=ni_to_ns[ni],
maintenances=node_mnts,
)
self.assertEqual(cv.get_node_view_by_node_index(ni), nv)
self.assertEqual(cv.get_node_name_by_node_index(ni), nn)
self.assertEqual(cv.get_node_config_by_node_index(ni), nc)
self.assertEqual(cv.get_node_state_by_node_index(ni), ns)
self.assertEqual(cv.get_node_maintenances_by_node_index(ni),
node_mnts)
self.assertEqual(cv.get_node_view_by_node_name(nn), nv)
self.assertEqual(cv.get_node_index_by_node_name(nn), ni)
self.assertEqual(cv.get_node_config_by_node_name(nn), nc)
self.assertEqual(cv.get_node_state_by_node_name(nn), ns)
self.assertEqual(cv.get_node_maintenances_by_node_name(nn),
node_mnts)
self.assertEqual(cv.get_node_view(node_name=nn), nv)
self.assertEqual(cv.get_node_index(node_name=nn), ni)
self.assertEqual(cv.get_node_config(node_name=nn), nc)
self.assertEqual(cv.get_node_state(node_name=nn), ns)
self.assertEqual(cv.get_node_maintenances(node_name=nn), node_mnts)
self.assertEqual(cv.get_node_view(node_index=ni), nv)
self.assertEqual(cv.get_node_name(node_index=ni), nn)
self.assertEqual(cv.get_node_config(node_index=ni), nc)
self.assertEqual(cv.get_node_state(node_index=ni), ns)
self.assertEqual(cv.get_node_maintenances(node_index=ni),
node_mnts)
with self.assertRaises(ValueError):
cv.get_node_view(None, None)
with self.assertRaises(ValueError):
cv.get_node_config(None, None)
with self.assertRaises(ValueError):
cv.get_node_state(None, None)
with self.assertRaises(ValueError):
cv.get_node_maintenances(None, None)
# mismatch node_index and node_name
if len(nis) > 1:
nn = ni_to_nc[nis[0]].name
ni = nis[1]
with self.assertRaises(ValueError):
cv.get_node_view(ni, nn)
with self.assertRaises(ValueError):
cv.get_node_config(ni, nn)
with self.assertRaises(ValueError):
cv.get_node_state(ni, nn)
with self.assertRaises(ValueError):
cv.get_node_maintenances(ni, nn)
# non-existent node_index
with self.assertRaises(NodeNotFoundError):
cv.get_node_view(node_index=max(nis) + 1)
# non-existent node_name
with self.assertRaises(NodeNotFoundError):
nns = {nc.name for nc in ncs}
nn: str = ""
while True:
nn = gen_word()
if nn not in nns:
break
cv.get_node_view(node_name=nn)
for mnt in mnts:
group_id: str = mnt.group_id or ""
self.assertEqual(cv.get_maintenance_by_id(group_id), mnt)
self.assertTupleEqual(
cv.get_node_indexes_by_maintenance_id(group_id),
tuple(
sorted(
set({
n.node_index
for n in mnt.sequencer_nodes
if n.node_index is not None
}).union({
s.node.node_index
for s in mnt.shards
if s.node.node_index is not None
}))),
)
self.assertEqual(group_id,
cv.get_maintenance_view_by_id(group_id).group_id)
self.assertListEqual(
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Generator[typing.Optional[str], None, None]`.
sorted(m.group_id for m in mnts),
sorted(mv.group_id for mv in cv.get_all_maintenance_views()),
)
# expand_shards
self.assertEqual(
cv.expand_shards(shards=[
ShardID(node=NodeID(node_index=nis[0]), shard_index=0)
]),
(ShardID(
node=NodeID(
node_index=ni_to_nc[nis[0]].node_index,
name=ni_to_nc[nis[0]].name,
address=ni_to_nc[nis[0]].data_address,
),
shard_index=0,
), ),
)
num_shards: int = getattr(ni_to_nc[nis[0]], "storage",
StorageConfig()).num_shards
self.assertEqual(
len(
cv.expand_shards(shards=[
ShardID(node=NodeID(node_index=nis[0]),
shard_index=ALL_SHARDS)
])),
num_shards,
)
self.assertEqual(
len(
cv.expand_shards(shards=[
ShardID(node=NodeID(node_index=nis[0]),
shard_index=ALL_SHARDS),
ShardID(node=NodeID(node_index=nis[0]),
shard_index=ALL_SHARDS),
ShardID(node=NodeID(node_index=nis[1]),
shard_index=ALL_SHARDS),
])),
getattr(ni_to_nc[nis[0]], "storage", StorageConfig()).num_shards +
getattr(ni_to_nc[nis[1]], "storage", StorageConfig()).num_shards,
)
self.assertEqual(
len(
cv.expand_shards(
shards=[
ShardID(node=NodeID(node_index=nis[0]),
shard_index=ALL_SHARDS),
ShardID(node=NodeID(node_index=nis[1]), shard_index=0),
],
node_ids=[NodeID(node_index=0)],
)),
getattr(ni_to_nc[nis[0]], "storage", StorageConfig()).num_shards +
1,
)
# normalize_node_id
self.assertEqual(
cv.normalize_node_id(NodeID(node_index=nis[0])),
NodeID(
node_index=nis[0],
address=ni_to_nc[nis[0]].data_address,
name=ni_to_nc[nis[0]].name,
),
)
self.assertEqual(
cv.normalize_node_id(NodeID(name=ni_to_nc[nis[0]].name)),
NodeID(
node_index=nis[0],
address=ni_to_nc[nis[0]].data_address,
name=ni_to_nc[nis[0]].name,
),
)
# get_node_id
self.assertEqual(cv.get_node_id(node_index=0).node_index, 0)
# search_maintenances
storages_node_views = [
nv for nv in cv.get_all_node_views() if nv.is_storage
]
sequencers_node_views = [
nv for nv in cv.get_all_node_views() if nv.is_sequencer
]
self.assertEqual(len(cv.search_maintenances()), len(mnts))
self.assertEqual(len(cv.search_maintenances(node_ids=[])), 0)
self.assertEqual(
len(
cv.search_maintenances(sequencer_nodes=[
storages_node_views[3].node_id,
sequencers_node_views[4].node_id,
])),
1,
)
self.assertEqual(
len(
cv.search_maintenances(node_ids=[
storages_node_views[3].node_id,
sequencers_node_views[4].node_id,
])),
1,
)
self.assertEqual(
len(
cv.search_maintenances(shards=[
ShardID(node=storages_node_views[1].node_id, shard_index=1)
])),
1,
)
# shard_target_state
self.assertEqual(
len(
cv.search_maintenances(
shard_target_state=ShardOperationalState.MAY_DISAPPEAR)),
3,
)
self.assertEqual(
len(
cv.search_maintenances(
shard_target_state=ShardOperationalState.DRAINED)),
0,
)
# sequencer_target_state
self.assertEqual(
len(
cv.search_maintenances(
sequencer_target_state=SequencingState.ENABLED)),
0,
)
self.assertEqual(
len(
cv.search_maintenances(
sequencer_target_state=SequencingState.DISABLED)),
3,
)
self.assertEqual(len(cv.search_maintenances(user="******")), 1)
self.assertEqual(len(cv.search_maintenances(reason="whatever")), 1)
self.assertEqual(len(cv.search_maintenances(skip_safety_checks=True)),
0)
self.assertEqual(len(cv.search_maintenances(skip_safety_checks=False)),
4)
self.assertEqual(
len(cv.search_maintenances(force_restore_rebuilding=True)), 0)
self.assertEqual(
len(cv.search_maintenances(force_restore_rebuilding=False)), 4)
self.assertEqual(
len(cv.search_maintenances(allow_passive_drains=True)), 0)
self.assertEqual(
len(cv.search_maintenances(allow_passive_drains=False)), 4)
self.assertEqual(
len(cv.search_maintenances(group_id=mnts[0].group_id)), 1)
self.assertEqual(
len(
cv.search_maintenances(
progress=MaintenanceProgress.IN_PROGRESS)), 4)