async def test_smoke(self):
ni = 0
async with MockAdminAPI() as client:
nodes_config_resp = await client.getNodesConfig(
NodesFilter(node=NodeID(node_index=ni))
)
nodes_state_resp = await client.getNodesState(
NodesStateRequest(filter=NodesFilter(node=NodeID(node_index=ni)))
)
maintenances_resp = await client.getMaintenances(MaintenancesFilter())
nc = nodes_config_resp.nodes[0]
ns = nodes_state_resp.states[0]
mnt_ids = set()
for mnt in maintenances_resp.maintenances:
for s in mnt.shards:
if s.node.node_index == ni:
mnt_ids.add(mnt.group_id)
for n in mnt.sequencer_nodes:
if n.node_index == ni:
mnt_ids.add(mnt.group_id)
mnts = tuple(
sorted(
(
mnt
for mnt in maintenances_resp.maintenances
if mnt.group_id in mnt_ids
),
key=operator.attrgetter("group_id"),
)
)
nv = NodeView(node_config=nc, node_state=ns, maintenances=mnts)
self._validate(nv, nc, ns, mnts)
def node_index_to_node_view(self) -> Dict[int, NodeView]:
if self._node_index_to_node_view is None:
self._node_index_to_node_view = {
ni: NodeView(
node_state=self.node_index_to_node_state[ni],
node_config=self.node_index_to_node_config[ni],
maintenances=self.node_index_to_maintenances[ni],
)
for ni in self.node_indexes
}
return self._node_index_to_node_view
def test_empty(self):
nc = NodeConfig(
node_index=0,
data_address=gen_SocketAddress(),
roles=set(),
location_per_scope={},
name="",
)
ns = NodeState(node_index=0)
mnts = []
self._validate(NodeView(nc, ns, mnts), nc, ns, mnts)
def test_unix_socket(self):
nc = NodeConfig(
node_index=0,
data_address=SocketAddress(address_family=SocketAddressFamily.UNIX,
path="/path/to/unix.sock").to_thrift(),
roles=set(),
location_per_scope={},
name="",
)
ns = NodeState(node_index=0)
mnts = []
self._validate(NodeView(nc, ns, mnts), nc, ns, mnts)
def _node_index_to_node_view(self) -> Dict[int, NodeView]:
if self._node_index_to_node_view_dict is None:
self._node_index_to_node_view_dict = {
ni: NodeView(
node_config=self._node_index_to_node_config[ni],
node_state=self._node_index_to_node_state[ni],
maintenances=self._node_index_to_maintenances[ni],
)
for ni in self._node_indexes
}
# pyre-fixme[7]: Expected `Dict[int, NodeView]` but got `Optional[Dict[int,
# NodeView]]`.
return self._node_index_to_node_view_dict
async def test_smoke(self):
ni = 0
async with MockAdminAPI() as client:
cv = await get_cluster_view(client)
maintenances_resp = await apply_maintenance(
client=client,
shards=[
ShardID(
node=cv.get_node_view_by_node_index(0).node_id, shard_index=1
)
],
sequencer_nodes=[cv.get_node_view_by_node_index(0).node_id],
)
(
nodes_config_resp,
nodes_state_resp,
maintenances_resp,
) = await asyncio.gather(
client.getNodesConfig(NodesFilter(node=NodeID(node_index=ni))),
client.getNodesState(
NodesStateRequest(filter=NodesFilter(node=NodeID(node_index=ni)))
),
client.getMaintenances(MaintenancesFilter()),
)
nc = [n for n in nodes_config_resp.nodes if n.node_index == ni][0]
ns = [n for n in nodes_state_resp.states if n.node_index == ni][0]
mnt_ids = set()
for mnt in maintenances_resp.maintenances:
for s in mnt.shards:
if s.node.node_index == ni:
mnt_ids.add(mnt.group_id)
for n in mnt.sequencer_nodes:
if n.node_index == ni:
mnt_ids.add(mnt.group_id)
mnts = tuple(
sorted(
(
mnt
for mnt in maintenances_resp.maintenances
if mnt.group_id in mnt_ids
),
key=operator.attrgetter("group_id"),
)
)
nv = NodeView(node_config=nc, node_state=ns, maintenances=mnts)
self._validate(nv, nc, ns, mnts)
def test_no_name(self):
addr = gen_SocketAddress()
self.assertEqual(
NodeView(
node_config=NodeConfig(
node_index=0,
data_address=addr,
roles=set(),
location_per_scope={},
name="",
),
node_state=NodeState(node_index=0),
maintenances=[],
).node_name,
str(SocketAddress.from_thrift(addr)),
)
async def test_mismatch(self):
async with MockAdminAPI() as client:
(
nodes_config_resp,
nodes_state_resp,
maintenances_resp,
) = await asyncio.gather(
client.getNodesConfig(NodesFilter(node=NodeID(node_index=0))),
client.getNodesState(
NodesStateRequest(filter=NodesFilter(node=NodeID(node_index=1)))
),
client.getMaintenances(MaintenancesFilter()),
)
with self.assertRaises(ValueError):
NodeView(
node_config=nodes_config_resp.nodes[0],
node_state=nodes_state_resp.states[0],
maintenances=maintenances_resp.maintenances,
)
def _validate(
self,
nv: NodeView,
nc: NodeConfig,
ns: NodeState,
mnts: Tuple[MaintenanceDefinition, ...],
):
self.assertEqual(nv.node_config, nc)
self.assertEqual(nv.node_state, ns)
self.assertEqual(nv.maintenances, mnts)
self.assertEqual(nv.node_index, nc.node_index)
if nc.name:
self.assertEqual(nv.node_name, nc.name)
else:
self.assertEqual(
nv.node_name, str(SocketAddress.from_thrift(nc.data_address))
)
self.assertEqual(nv.data_address, SocketAddress.from_thrift(nc.data_address))
if nv.thrift_address.address_family == SocketAddressFamily.INET:
assert nv.thrift_address.address is not None
from_nc = SocketAddress.from_thrift(nc.data_address)
assert from_nc.address is not None
self.assertEqual(nv.thrift_address.port, 6440)
self.assertEqual(
nv.thrift_address.address.compressed, from_nc.address.compressed
)
self.assertEqual(
nv.node_id,
NodeID(node_index=nc.node_index, address=nc.data_address, name=nc.name),
)
self.assertEqual(nv.location, nc.location)
self.assertEqual(nv.location_per_scope, nc.location_per_scope)
self.assertEqual(nv.roles, nc.roles)
for r in Role:
self.assertEqual(nv.has_role(r), r in nc.roles)
self.assertEqual(nv.is_sequencer, Role.SEQUENCER in nc.roles)
self.assertEqual(nv.is_storage, Role.STORAGE in nc.roles)
self.assertEqual(nv.daemon_state, ns.daemon_state)
if Role.SEQUENCER in nc.roles:
assert nc.sequencer is not None
self.assertEqual(nv.sequencer_config, nc.sequencer)
self.assertEqual(nv.sequencer_weight, nc.sequencer.weight)
assert ns.sequencer_state is not None
self.assertEqual(nv.sequencer_state, ns.sequencer_state)
self.assertEqual(nv.sequencing_state, ns.sequencer_state.state)
else:
self.assertIsNone(nv.sequencer_config)
self.assertIsNone(nv.sequencer_state)
self.assertIsNone(nv.sequencer_weight)
self.assertIsNone(nv.sequencing_state)
if Role.STORAGE in nc.roles:
assert nc.storage is not None
assert ns.shard_states is not None
self.assertEqual(nv.storage_config, nc.storage)
self.assertEqual(nv.storage_weight, nc.storage.weight)
self.assertEqual(nv.num_shards, nc.storage.num_shards)
self.assertEqual(nv.shard_states, ns.shard_states)
self.assertListEqual(
nv.shards_data_health, [s.data_health for s in ns.shard_states]
)
self.assertEqual(
nv.shards_data_health_count,
Counter(s.data_health for s in ns.shard_states),
)
self.assertListEqual(
nv.shards_current_storage_state,
[s.current_storage_state for s in ns.shard_states],
)
self.assertEqual(
nv.shards_current_storage_state_count,
Counter(s.current_storage_state for s in ns.shard_states),
)
self.assertListEqual(
nv.shards_current_operational_state,
[s.current_operational_state for s in ns.shard_states],
)
self.assertEqual(
nv.shards_current_operational_state_count,
Counter(s.current_operational_state for s in ns.shard_states),
)
self.assertListEqual(
nv.shards_membership_storage_state,
[s.storage_state for s in ns.shard_states],
)
self.assertEqual(
nv.shards_membership_storage_state_count,
Counter(s.storage_state for s in ns.shard_states),
)
self.assertListEqual(
nv.shards_metadata_state, [s.metadata_state for s in ns.shard_states]
)
self.assertEqual(
nv.shards_metadata_state_count,
Counter(s.metadata_state for s in ns.shard_states),
)
else:
self.assertIsNone(nv.storage_config)
self.assertIsNone(nv.storage_weight)
self.assertIsNone(nv.num_shards)
self.assertEqual(nv.shard_states, [])
def _validate(
self,
cv: ClusterView,
ncs: List[NodeConfig],
nss: List[NodeState],
mnts: Tuple[MaintenanceDefinition, ...],
):
nis = sorted(nc.node_index for nc in ncs)
ni_to_nc = {nc.node_index: nc for nc in ncs}
ni_to_ns = {ns.node_index: ns for ns in nss}
ni_to_mnts: Dict[int,
List[MaintenanceDefinition]] = {ni: []
for ni in nis}
for mnt in mnts:
mnt_nis = set()
for s in mnt.shards:
assert s.node.node_index is not None
mnt_nis.add(s.node.node_index)
for n in mnt.sequencer_nodes:
assert n.node_index is not None
mnt_nis.add(n.node_index)
for ni in mnt_nis:
ni_to_mnts[ni].append(mnt)
self.assertEqual(sorted(cv.get_all_node_indexes()),
sorted(ni_to_nc.keys()))
self.assertEqual(
sorted(cv.get_all_node_views(),
key=operator.attrgetter("node_index")),
sorted(
(NodeView(
node_config=ni_to_nc[ni],
node_state=ni_to_ns[ni],
maintenances=tuple(ni_to_mnts[ni]),
) for ni in ni_to_nc.keys()),
key=operator.attrgetter("node_index"),
),
)
self.assertEqual(sorted(cv.get_all_node_names()),
sorted(nc.name for nc in ncs))
self.assertEqual(sorted(cv.get_all_maintenance_ids()),
sorted(mnt.group_id for mnt in mnts))
self.assertEqual(
sorted(cv.get_all_maintenances(),
key=operator.attrgetter("group_id")),
sorted(mnts, key=operator.attrgetter("group_id")),
)
for ni in nis:
nn = ni_to_nc[ni].name
nc = ni_to_nc[ni]
ns = ni_to_ns[ni]
mnts = tuple(ni_to_mnts[ni])
nv = NodeView(node_config=ni_to_nc[ni],
node_state=ni_to_ns[ni],
maintenances=mnts)
self.assertEqual(cv.get_node_view_by_node_index(ni), nv)
self.assertEqual(cv.get_node_name_by_node_index(ni), nn)
self.assertEqual(cv.get_node_config_by_node_index(ni), nc)
self.assertEqual(cv.get_node_state_by_node_index(ni), ns)
self.assertEqual(cv.get_maintenances_by_node_index(ni), mnts)
self.assertEqual(cv.get_node_view_by_node_name(nn), nv)
self.assertEqual(cv.get_node_index_by_node_name(nn), ni)
self.assertEqual(cv.get_node_config_by_node_name(nn), nc)
self.assertEqual(cv.get_node_state_by_node_name(nn), ns)
self.assertEqual(cv.get_maintenances_by_node_name(nn), mnts)
self.assertEqual(cv.get_node_view(node_name=nn), nv)
self.assertEqual(cv.get_node_index(node_name=nn), ni)
self.assertEqual(cv.get_node_config(node_name=nn), nc)
self.assertEqual(cv.get_node_state(node_name=nn), ns)
self.assertEqual(cv.get_maintenances(node_name=nn), mnts)
self.assertEqual(cv.get_node_view(node_index=ni), nv)
self.assertEqual(cv.get_node_name(node_index=ni), nn)
self.assertEqual(cv.get_node_config(node_index=ni), nc)
self.assertEqual(cv.get_node_state(node_index=ni), ns)
self.assertEqual(cv.get_maintenances(node_index=ni), mnts)
with self.assertRaises(ValueError):
cv.get_node_view(None, None)
with self.assertRaises(ValueError):
cv.get_node_config(None, None)
with self.assertRaises(ValueError):
cv.get_node_state(None, None)
with self.assertRaises(ValueError):
cv.get_maintenances(None, None)
# mismatch node_index and node_name
if len(nis) > 1:
nn = ni_to_nc[nis[0]].name
ni = nis[1]
with self.assertRaises(ValueError):
cv.get_node_view(ni, nn)
with self.assertRaises(ValueError):
cv.get_node_config(ni, nn)
with self.assertRaises(ValueError):
cv.get_node_state(ni, nn)
with self.assertRaises(ValueError):
cv.get_maintenances(ni, nn)
# non-existent node_index
with self.assertRaises(KeyError):
cv.get_node_view(node_index=max(nis) + 1)
# non-existent node_name
with self.assertRaises(KeyError):
nns = {nc.name for nc in ncs}
while True:
nn = gen_word()
if nn not in nns:
break
cv.get_node_view(node_name=nn)
for mnt in mnts:
assert mnt.group_id is not None
self.assertEqual(cv.get_maintenance_by_id(mnt.group_id), mnt)
def _validate(
self,
cv: ClusterView,
ncs: List[NodeConfig],
nss: List[NodeState],
mnts: Tuple[MaintenanceDefinition, ...],
) -> None:
nis = sorted(nc.node_index for nc in ncs)
ni_to_nc = {nc.node_index: nc for nc in ncs}
ni_to_ns = {ns.node_index: ns for ns in nss}
ni_to_mnts: Dict[int,
List[MaintenanceDefinition]] = {ni: []
for ni in nis}
for mnt in mnts:
mnt_nis = set()
for s in mnt.shards:
assert s.node.node_index is not None
mnt_nis.add(s.node.node_index)
for n in mnt.sequencer_nodes:
assert n.node_index is not None
mnt_nis.add(n.node_index)
for ni in mnt_nis:
ni_to_mnts[ni].append(mnt)
self.assertEqual(sorted(cv.get_all_node_indexes()),
sorted(ni_to_nc.keys()))
self.assertEqual(
sorted(cv.get_all_node_views(),
key=operator.attrgetter("node_index")),
sorted(
(NodeView(
node_config=ni_to_nc[ni],
node_state=ni_to_ns[ni],
maintenances=tuple(ni_to_mnts[ni]),
) for ni in ni_to_nc.keys()),
key=operator.attrgetter("node_index"),
),
)
self.assertEqual(sorted(cv.get_all_node_names()),
sorted(nc.name for nc in ncs))
self.assertEqual(
sorted(cv.get_all_maintenance_ids()),
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Generator[typing.Optional[str], None, None]`.
sorted(mnt.group_id for mnt in mnts),
)
self.assertEqual(
sorted(cv.get_all_maintenances(),
key=operator.attrgetter("group_id")),
sorted(mnts, key=operator.attrgetter("group_id")),
)
for ni in nis:
nn = ni_to_nc[ni].name
nc = ni_to_nc[ni]
ns = ni_to_ns[ni]
node_mnts = tuple(ni_to_mnts[ni])
nv = NodeView(
node_config=ni_to_nc[ni],
node_state=ni_to_ns[ni],
maintenances=node_mnts,
)
self.assertEqual(cv.get_node_view_by_node_index(ni), nv)
self.assertEqual(cv.get_node_name_by_node_index(ni), nn)
self.assertEqual(cv.get_node_config_by_node_index(ni), nc)
self.assertEqual(cv.get_node_state_by_node_index(ni), ns)
self.assertEqual(cv.get_node_maintenances_by_node_index(ni),
node_mnts)
self.assertEqual(cv.get_node_view_by_node_name(nn), nv)
self.assertEqual(cv.get_node_index_by_node_name(nn), ni)
self.assertEqual(cv.get_node_config_by_node_name(nn), nc)
self.assertEqual(cv.get_node_state_by_node_name(nn), ns)
self.assertEqual(cv.get_node_maintenances_by_node_name(nn),
node_mnts)
self.assertEqual(cv.get_node_view(node_name=nn), nv)
self.assertEqual(cv.get_node_index(node_name=nn), ni)
self.assertEqual(cv.get_node_config(node_name=nn), nc)
self.assertEqual(cv.get_node_state(node_name=nn), ns)
self.assertEqual(cv.get_node_maintenances(node_name=nn), node_mnts)
self.assertEqual(cv.get_node_view(node_index=ni), nv)
self.assertEqual(cv.get_node_name(node_index=ni), nn)
self.assertEqual(cv.get_node_config(node_index=ni), nc)
self.assertEqual(cv.get_node_state(node_index=ni), ns)
self.assertEqual(cv.get_node_maintenances(node_index=ni),
node_mnts)
with self.assertRaises(ValueError):
cv.get_node_view(None, None)
with self.assertRaises(ValueError):
cv.get_node_config(None, None)
with self.assertRaises(ValueError):
cv.get_node_state(None, None)
with self.assertRaises(ValueError):
cv.get_node_maintenances(None, None)
# mismatch node_index and node_name
if len(nis) > 1:
nn = ni_to_nc[nis[0]].name
ni = nis[1]
with self.assertRaises(ValueError):
cv.get_node_view(ni, nn)
with self.assertRaises(ValueError):
cv.get_node_config(ni, nn)
with self.assertRaises(ValueError):
cv.get_node_state(ni, nn)
with self.assertRaises(ValueError):
cv.get_node_maintenances(ni, nn)
# non-existent node_index
with self.assertRaises(NodeNotFoundError):
cv.get_node_view(node_index=max(nis) + 1)
# non-existent node_name
with self.assertRaises(NodeNotFoundError):
nns = {nc.name for nc in ncs}
nn: str = ""
while True:
nn = gen_word()
if nn not in nns:
break
cv.get_node_view(node_name=nn)
for mnt in mnts:
group_id: str = mnt.group_id or ""
self.assertEqual(cv.get_maintenance_by_id(group_id), mnt)
self.assertTupleEqual(
cv.get_node_indexes_by_maintenance_id(group_id),
tuple(
sorted(
set({
n.node_index
for n in mnt.sequencer_nodes
if n.node_index is not None
}).union({
s.node.node_index
for s in mnt.shards
if s.node.node_index is not None
}))),
)
self.assertEqual(group_id,
cv.get_maintenance_view_by_id(group_id).group_id)
self.assertListEqual(
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Generator[typing.Optional[str], None, None]`.
sorted(m.group_id for m in mnts),
sorted(mv.group_id for mv in cv.get_all_maintenance_views()),
)
# expand_shards
self.assertEqual(
cv.expand_shards(shards=[
ShardID(node=NodeID(node_index=nis[0]), shard_index=0)
]),
(ShardID(
node=NodeID(
node_index=ni_to_nc[nis[0]].node_index,
name=ni_to_nc[nis[0]].name,
address=ni_to_nc[nis[0]].data_address,
),
shard_index=0,
), ),
)
num_shards: int = getattr(ni_to_nc[nis[0]], "storage",
StorageConfig()).num_shards
self.assertEqual(
len(
cv.expand_shards(shards=[
ShardID(node=NodeID(node_index=nis[0]),
shard_index=ALL_SHARDS)
])),
num_shards,
)
self.assertEqual(
len(
cv.expand_shards(shards=[
ShardID(node=NodeID(node_index=nis[0]),
shard_index=ALL_SHARDS),
ShardID(node=NodeID(node_index=nis[0]),
shard_index=ALL_SHARDS),
ShardID(node=NodeID(node_index=nis[1]),
shard_index=ALL_SHARDS),
])),
getattr(ni_to_nc[nis[0]], "storage", StorageConfig()).num_shards +
getattr(ni_to_nc[nis[1]], "storage", StorageConfig()).num_shards,
)
self.assertEqual(
len(
cv.expand_shards(
shards=[
ShardID(node=NodeID(node_index=nis[0]),
shard_index=ALL_SHARDS),
ShardID(node=NodeID(node_index=nis[1]), shard_index=0),
],
node_ids=[NodeID(node_index=0)],
)),
getattr(ni_to_nc[nis[0]], "storage", StorageConfig()).num_shards +
1,
)
# normalize_node_id
self.assertEqual(
cv.normalize_node_id(NodeID(node_index=nis[0])),
NodeID(
node_index=nis[0],
address=ni_to_nc[nis[0]].data_address,
name=ni_to_nc[nis[0]].name,
),
)
self.assertEqual(
cv.normalize_node_id(NodeID(name=ni_to_nc[nis[0]].name)),
NodeID(
node_index=nis[0],
address=ni_to_nc[nis[0]].data_address,
name=ni_to_nc[nis[0]].name,
),
)
# get_node_id
self.assertEqual(cv.get_node_id(node_index=0).node_index, 0)
# search_maintenances
storages_node_views = [
nv for nv in cv.get_all_node_views() if nv.is_storage
]
sequencers_node_views = [
nv for nv in cv.get_all_node_views() if nv.is_sequencer
]
self.assertEqual(len(cv.search_maintenances()), len(mnts))
self.assertEqual(len(cv.search_maintenances(node_ids=[])), 0)
self.assertEqual(
len(
cv.search_maintenances(sequencer_nodes=[
storages_node_views[3].node_id,
sequencers_node_views[4].node_id,
])),
1,
)
self.assertEqual(
len(
cv.search_maintenances(node_ids=[
storages_node_views[3].node_id,
sequencers_node_views[4].node_id,
])),
1,
)
self.assertEqual(
len(
cv.search_maintenances(shards=[
ShardID(node=storages_node_views[1].node_id, shard_index=1)
])),
1,
)
# shard_target_state
self.assertEqual(
len(
cv.search_maintenances(
shard_target_state=ShardOperationalState.MAY_DISAPPEAR)),
3,
)
self.assertEqual(
len(
cv.search_maintenances(
shard_target_state=ShardOperationalState.DRAINED)),
0,
)
# sequencer_target_state
self.assertEqual(
len(
cv.search_maintenances(
sequencer_target_state=SequencingState.ENABLED)),
0,
)
self.assertEqual(
len(
cv.search_maintenances(
sequencer_target_state=SequencingState.DISABLED)),
3,
)
self.assertEqual(len(cv.search_maintenances(user="******")), 1)
self.assertEqual(len(cv.search_maintenances(reason="whatever")), 1)
self.assertEqual(len(cv.search_maintenances(skip_safety_checks=True)),
0)
self.assertEqual(len(cv.search_maintenances(skip_safety_checks=False)),
4)
self.assertEqual(
len(cv.search_maintenances(force_restore_rebuilding=True)), 0)
self.assertEqual(
len(cv.search_maintenances(force_restore_rebuilding=False)), 4)
self.assertEqual(
len(cv.search_maintenances(allow_passive_drains=True)), 0)
self.assertEqual(
len(cv.search_maintenances(allow_passive_drains=False)), 4)
self.assertEqual(
len(cv.search_maintenances(group_id=mnts[0].group_id)), 1)
self.assertEqual(
len(
cv.search_maintenances(
progress=MaintenanceProgress.IN_PROGRESS)), 4)