async def test_smoke(self): ni = 0 async with MockAdminAPI() as client: nodes_config_resp = await client.getNodesConfig( NodesFilter(node=NodeID(node_index=ni)) ) nodes_state_resp = await client.getNodesState( NodesStateRequest(filter=NodesFilter(node=NodeID(node_index=ni))) ) maintenances_resp = await client.getMaintenances(MaintenancesFilter()) nc = nodes_config_resp.nodes[0] ns = nodes_state_resp.states[0] mnt_ids = set() for mnt in maintenances_resp.maintenances: for s in mnt.shards: if s.node.node_index == ni: mnt_ids.add(mnt.group_id) for n in mnt.sequencer_nodes: if n.node_index == ni: mnt_ids.add(mnt.group_id) mnts = tuple( sorted( ( mnt for mnt in maintenances_resp.maintenances if mnt.group_id in mnt_ids ), key=operator.attrgetter("group_id"), ) ) nv = NodeView(node_config=nc, node_state=ns, maintenances=mnts) self._validate(nv, nc, ns, mnts)
def node_index_to_node_view(self) -> Dict[int, NodeView]: if self._node_index_to_node_view is None: self._node_index_to_node_view = { ni: NodeView( node_state=self.node_index_to_node_state[ni], node_config=self.node_index_to_node_config[ni], maintenances=self.node_index_to_maintenances[ni], ) for ni in self.node_indexes } return self._node_index_to_node_view
def test_empty(self): nc = NodeConfig( node_index=0, data_address=gen_SocketAddress(), roles=set(), location_per_scope={}, name="", ) ns = NodeState(node_index=0) mnts = [] self._validate(NodeView(nc, ns, mnts), nc, ns, mnts)
def test_unix_socket(self): nc = NodeConfig( node_index=0, data_address=SocketAddress(address_family=SocketAddressFamily.UNIX, path="/path/to/unix.sock").to_thrift(), roles=set(), location_per_scope={}, name="", ) ns = NodeState(node_index=0) mnts = [] self._validate(NodeView(nc, ns, mnts), nc, ns, mnts)
def _node_index_to_node_view(self) -> Dict[int, NodeView]: if self._node_index_to_node_view_dict is None: self._node_index_to_node_view_dict = { ni: NodeView( node_config=self._node_index_to_node_config[ni], node_state=self._node_index_to_node_state[ni], maintenances=self._node_index_to_maintenances[ni], ) for ni in self._node_indexes } # pyre-fixme[7]: Expected `Dict[int, NodeView]` but got `Optional[Dict[int, # NodeView]]`. return self._node_index_to_node_view_dict
async def test_smoke(self): ni = 0 async with MockAdminAPI() as client: cv = await get_cluster_view(client) maintenances_resp = await apply_maintenance( client=client, shards=[ ShardID( node=cv.get_node_view_by_node_index(0).node_id, shard_index=1 ) ], sequencer_nodes=[cv.get_node_view_by_node_index(0).node_id], ) ( nodes_config_resp, nodes_state_resp, maintenances_resp, ) = await asyncio.gather( client.getNodesConfig(NodesFilter(node=NodeID(node_index=ni))), client.getNodesState( NodesStateRequest(filter=NodesFilter(node=NodeID(node_index=ni))) ), client.getMaintenances(MaintenancesFilter()), ) nc = [n for n in nodes_config_resp.nodes if n.node_index == ni][0] ns = [n for n in nodes_state_resp.states if n.node_index == ni][0] mnt_ids = set() for mnt in maintenances_resp.maintenances: for s in mnt.shards: if s.node.node_index == ni: mnt_ids.add(mnt.group_id) for n in mnt.sequencer_nodes: if n.node_index == ni: mnt_ids.add(mnt.group_id) mnts = tuple( sorted( ( mnt for mnt in maintenances_resp.maintenances if mnt.group_id in mnt_ids ), key=operator.attrgetter("group_id"), ) ) nv = NodeView(node_config=nc, node_state=ns, maintenances=mnts) self._validate(nv, nc, ns, mnts)
def test_no_name(self): addr = gen_SocketAddress() self.assertEqual( NodeView( node_config=NodeConfig( node_index=0, data_address=addr, roles=set(), location_per_scope={}, name="", ), node_state=NodeState(node_index=0), maintenances=[], ).node_name, str(SocketAddress.from_thrift(addr)), )
async def test_mismatch(self): async with MockAdminAPI() as client: ( nodes_config_resp, nodes_state_resp, maintenances_resp, ) = await asyncio.gather( client.getNodesConfig(NodesFilter(node=NodeID(node_index=0))), client.getNodesState( NodesStateRequest(filter=NodesFilter(node=NodeID(node_index=1))) ), client.getMaintenances(MaintenancesFilter()), ) with self.assertRaises(ValueError): NodeView( node_config=nodes_config_resp.nodes[0], node_state=nodes_state_resp.states[0], maintenances=maintenances_resp.maintenances, )
def _validate( self, cv: ClusterView, ncs: List[NodeConfig], nss: List[NodeState], mnts: Tuple[MaintenanceDefinition, ...], ): nis = sorted(nc.node_index for nc in ncs) ni_to_nc = {nc.node_index: nc for nc in ncs} ni_to_ns = {ns.node_index: ns for ns in nss} ni_to_mnts: Dict[int, List[MaintenanceDefinition]] = {ni: [] for ni in nis} for mnt in mnts: mnt_nis = set() for s in mnt.shards: assert s.node.node_index is not None mnt_nis.add(s.node.node_index) for n in mnt.sequencer_nodes: assert n.node_index is not None mnt_nis.add(n.node_index) for ni in mnt_nis: ni_to_mnts[ni].append(mnt) self.assertEqual(sorted(cv.get_all_node_indexes()), sorted(ni_to_nc.keys())) self.assertEqual( sorted(cv.get_all_node_views(), key=operator.attrgetter("node_index")), sorted( (NodeView( node_config=ni_to_nc[ni], node_state=ni_to_ns[ni], maintenances=tuple(ni_to_mnts[ni]), ) for ni in ni_to_nc.keys()), key=operator.attrgetter("node_index"), ), ) self.assertEqual(sorted(cv.get_all_node_names()), sorted(nc.name for nc in ncs)) self.assertEqual(sorted(cv.get_all_maintenance_ids()), sorted(mnt.group_id for mnt in mnts)) self.assertEqual( sorted(cv.get_all_maintenances(), key=operator.attrgetter("group_id")), sorted(mnts, key=operator.attrgetter("group_id")), ) for ni in nis: nn = ni_to_nc[ni].name nc = ni_to_nc[ni] ns = ni_to_ns[ni] mnts = tuple(ni_to_mnts[ni]) nv = NodeView(node_config=ni_to_nc[ni], node_state=ni_to_ns[ni], maintenances=mnts) self.assertEqual(cv.get_node_view_by_node_index(ni), nv) self.assertEqual(cv.get_node_name_by_node_index(ni), nn) self.assertEqual(cv.get_node_config_by_node_index(ni), nc) self.assertEqual(cv.get_node_state_by_node_index(ni), ns) self.assertEqual(cv.get_maintenances_by_node_index(ni), mnts) self.assertEqual(cv.get_node_view_by_node_name(nn), nv) self.assertEqual(cv.get_node_index_by_node_name(nn), ni) self.assertEqual(cv.get_node_config_by_node_name(nn), nc) self.assertEqual(cv.get_node_state_by_node_name(nn), ns) self.assertEqual(cv.get_maintenances_by_node_name(nn), mnts) self.assertEqual(cv.get_node_view(node_name=nn), nv) self.assertEqual(cv.get_node_index(node_name=nn), ni) self.assertEqual(cv.get_node_config(node_name=nn), nc) self.assertEqual(cv.get_node_state(node_name=nn), ns) self.assertEqual(cv.get_maintenances(node_name=nn), mnts) self.assertEqual(cv.get_node_view(node_index=ni), nv) self.assertEqual(cv.get_node_name(node_index=ni), nn) self.assertEqual(cv.get_node_config(node_index=ni), nc) self.assertEqual(cv.get_node_state(node_index=ni), ns) self.assertEqual(cv.get_maintenances(node_index=ni), mnts) with self.assertRaises(ValueError): cv.get_node_view(None, None) with self.assertRaises(ValueError): cv.get_node_config(None, None) with self.assertRaises(ValueError): cv.get_node_state(None, None) with self.assertRaises(ValueError): cv.get_maintenances(None, None) # mismatch node_index and node_name if len(nis) > 1: nn = ni_to_nc[nis[0]].name ni = nis[1] with self.assertRaises(ValueError): cv.get_node_view(ni, nn) with self.assertRaises(ValueError): cv.get_node_config(ni, nn) with self.assertRaises(ValueError): cv.get_node_state(ni, nn) with self.assertRaises(ValueError): cv.get_maintenances(ni, nn) # non-existent node_index with self.assertRaises(KeyError): cv.get_node_view(node_index=max(nis) + 1) # non-existent node_name with self.assertRaises(KeyError): nns = {nc.name for nc in ncs} while True: nn = gen_word() if nn not in nns: break cv.get_node_view(node_name=nn) for mnt in mnts: assert mnt.group_id is not None self.assertEqual(cv.get_maintenance_by_id(mnt.group_id), mnt)
def _validate( self, cv: ClusterView, ncs: List[NodeConfig], nss: List[NodeState], mnts: Tuple[MaintenanceDefinition, ...], ) -> None: nis = sorted(nc.node_index for nc in ncs) ni_to_nc = {nc.node_index: nc for nc in ncs} ni_to_ns = {ns.node_index: ns for ns in nss} ni_to_mnts: Dict[int, List[MaintenanceDefinition]] = {ni: [] for ni in nis} for mnt in mnts: mnt_nis = set() for s in mnt.shards: assert s.node.node_index is not None mnt_nis.add(s.node.node_index) for n in mnt.sequencer_nodes: assert n.node_index is not None mnt_nis.add(n.node_index) for ni in mnt_nis: ni_to_mnts[ni].append(mnt) self.assertEqual(sorted(cv.get_all_node_indexes()), sorted(ni_to_nc.keys())) self.assertEqual( sorted(cv.get_all_node_views(), key=operator.attrgetter("node_index")), sorted( (NodeView( node_config=ni_to_nc[ni], node_state=ni_to_ns[ni], maintenances=tuple(ni_to_mnts[ni]), ) for ni in ni_to_nc.keys()), key=operator.attrgetter("node_index"), ), ) self.assertEqual(sorted(cv.get_all_node_names()), sorted(nc.name for nc in ncs)) self.assertEqual( sorted(cv.get_all_maintenance_ids()), # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got # `Generator[typing.Optional[str], None, None]`. sorted(mnt.group_id for mnt in mnts), ) self.assertEqual( sorted(cv.get_all_maintenances(), key=operator.attrgetter("group_id")), sorted(mnts, key=operator.attrgetter("group_id")), ) for ni in nis: nn = ni_to_nc[ni].name nc = ni_to_nc[ni] ns = ni_to_ns[ni] node_mnts = tuple(ni_to_mnts[ni]) nv = NodeView( node_config=ni_to_nc[ni], node_state=ni_to_ns[ni], maintenances=node_mnts, ) self.assertEqual(cv.get_node_view_by_node_index(ni), nv) self.assertEqual(cv.get_node_name_by_node_index(ni), nn) self.assertEqual(cv.get_node_config_by_node_index(ni), nc) self.assertEqual(cv.get_node_state_by_node_index(ni), ns) self.assertEqual(cv.get_node_maintenances_by_node_index(ni), node_mnts) self.assertEqual(cv.get_node_view_by_node_name(nn), nv) self.assertEqual(cv.get_node_index_by_node_name(nn), ni) self.assertEqual(cv.get_node_config_by_node_name(nn), nc) self.assertEqual(cv.get_node_state_by_node_name(nn), ns) self.assertEqual(cv.get_node_maintenances_by_node_name(nn), node_mnts) self.assertEqual(cv.get_node_view(node_name=nn), nv) self.assertEqual(cv.get_node_index(node_name=nn), ni) self.assertEqual(cv.get_node_config(node_name=nn), nc) self.assertEqual(cv.get_node_state(node_name=nn), ns) self.assertEqual(cv.get_node_maintenances(node_name=nn), node_mnts) self.assertEqual(cv.get_node_view(node_index=ni), nv) self.assertEqual(cv.get_node_name(node_index=ni), nn) self.assertEqual(cv.get_node_config(node_index=ni), nc) self.assertEqual(cv.get_node_state(node_index=ni), ns) self.assertEqual(cv.get_node_maintenances(node_index=ni), node_mnts) with self.assertRaises(ValueError): cv.get_node_view(None, None) with self.assertRaises(ValueError): cv.get_node_config(None, None) with self.assertRaises(ValueError): cv.get_node_state(None, None) with self.assertRaises(ValueError): cv.get_node_maintenances(None, None) # mismatch node_index and node_name if len(nis) > 1: nn = ni_to_nc[nis[0]].name ni = nis[1] with self.assertRaises(ValueError): cv.get_node_view(ni, nn) with self.assertRaises(ValueError): cv.get_node_config(ni, nn) with self.assertRaises(ValueError): cv.get_node_state(ni, nn) with self.assertRaises(ValueError): cv.get_node_maintenances(ni, nn) # non-existent node_index with self.assertRaises(NodeNotFoundError): cv.get_node_view(node_index=max(nis) + 1) # non-existent node_name with self.assertRaises(NodeNotFoundError): nns = {nc.name for nc in ncs} nn: str = "" while True: nn = gen_word() if nn not in nns: break cv.get_node_view(node_name=nn) for mnt in mnts: group_id: str = mnt.group_id or "" self.assertEqual(cv.get_maintenance_by_id(group_id), mnt) self.assertTupleEqual( cv.get_node_indexes_by_maintenance_id(group_id), tuple( sorted( set({ n.node_index for n in mnt.sequencer_nodes if n.node_index is not None }).union({ s.node.node_index for s in mnt.shards if s.node.node_index is not None }))), ) self.assertEqual(group_id, cv.get_maintenance_view_by_id(group_id).group_id) self.assertListEqual( # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got # `Generator[typing.Optional[str], None, None]`. sorted(m.group_id for m in mnts), sorted(mv.group_id for mv in cv.get_all_maintenance_views()), ) # expand_shards self.assertEqual( cv.expand_shards(shards=[ ShardID(node=NodeID(node_index=nis[0]), shard_index=0) ]), (ShardID( node=NodeID( node_index=ni_to_nc[nis[0]].node_index, name=ni_to_nc[nis[0]].name, address=ni_to_nc[nis[0]].data_address, ), shard_index=0, ), ), ) num_shards: int = getattr(ni_to_nc[nis[0]], "storage", StorageConfig()).num_shards self.assertEqual( len( cv.expand_shards(shards=[ ShardID(node=NodeID(node_index=nis[0]), shard_index=ALL_SHARDS) ])), num_shards, ) self.assertEqual( len( cv.expand_shards(shards=[ ShardID(node=NodeID(node_index=nis[0]), shard_index=ALL_SHARDS), ShardID(node=NodeID(node_index=nis[0]), shard_index=ALL_SHARDS), ShardID(node=NodeID(node_index=nis[1]), shard_index=ALL_SHARDS), ])), getattr(ni_to_nc[nis[0]], "storage", StorageConfig()).num_shards + getattr(ni_to_nc[nis[1]], "storage", StorageConfig()).num_shards, ) self.assertEqual( len( cv.expand_shards( shards=[ ShardID(node=NodeID(node_index=nis[0]), shard_index=ALL_SHARDS), ShardID(node=NodeID(node_index=nis[1]), shard_index=0), ], node_ids=[NodeID(node_index=0)], )), getattr(ni_to_nc[nis[0]], "storage", StorageConfig()).num_shards + 1, ) # normalize_node_id self.assertEqual( cv.normalize_node_id(NodeID(node_index=nis[0])), NodeID( node_index=nis[0], address=ni_to_nc[nis[0]].data_address, name=ni_to_nc[nis[0]].name, ), ) self.assertEqual( cv.normalize_node_id(NodeID(name=ni_to_nc[nis[0]].name)), NodeID( node_index=nis[0], address=ni_to_nc[nis[0]].data_address, name=ni_to_nc[nis[0]].name, ), ) # get_node_id self.assertEqual(cv.get_node_id(node_index=0).node_index, 0) # search_maintenances storages_node_views = [ nv for nv in cv.get_all_node_views() if nv.is_storage ] sequencers_node_views = [ nv for nv in cv.get_all_node_views() if nv.is_sequencer ] self.assertEqual(len(cv.search_maintenances()), len(mnts)) self.assertEqual(len(cv.search_maintenances(node_ids=[])), 0) self.assertEqual( len( cv.search_maintenances(sequencer_nodes=[ storages_node_views[3].node_id, sequencers_node_views[4].node_id, ])), 1, ) self.assertEqual( len( cv.search_maintenances(node_ids=[ storages_node_views[3].node_id, sequencers_node_views[4].node_id, ])), 1, ) self.assertEqual( len( cv.search_maintenances(shards=[ ShardID(node=storages_node_views[1].node_id, shard_index=1) ])), 1, ) # shard_target_state self.assertEqual( len( cv.search_maintenances( shard_target_state=ShardOperationalState.MAY_DISAPPEAR)), 3, ) self.assertEqual( len( cv.search_maintenances( shard_target_state=ShardOperationalState.DRAINED)), 0, ) # sequencer_target_state self.assertEqual( len( cv.search_maintenances( sequencer_target_state=SequencingState.ENABLED)), 0, ) self.assertEqual( len( cv.search_maintenances( sequencer_target_state=SequencingState.DISABLED)), 3, ) self.assertEqual(len(cv.search_maintenances(user="******")), 1) self.assertEqual(len(cv.search_maintenances(reason="whatever")), 1) self.assertEqual(len(cv.search_maintenances(skip_safety_checks=True)), 0) self.assertEqual(len(cv.search_maintenances(skip_safety_checks=False)), 4) self.assertEqual( len(cv.search_maintenances(force_restore_rebuilding=True)), 0) self.assertEqual( len(cv.search_maintenances(force_restore_rebuilding=False)), 4) self.assertEqual( len(cv.search_maintenances(allow_passive_drains=True)), 0) self.assertEqual( len(cv.search_maintenances(allow_passive_drains=False)), 4) self.assertEqual( len(cv.search_maintenances(group_id=mnts[0].group_id)), 1) self.assertEqual( len( cv.search_maintenances( progress=MaintenanceProgress.IN_PROGRESS)), 4)