def impact(mv: MaintenanceView, show_safety_check_results: bool) -> str: if (mv.overall_status == MaintenanceOverallStatus.BLOCKED and show_safety_check_results): response = mv.last_check_impact_result shards = [] for ni in mv.affected_node_indexes: shards.extend(mv.get_shards_by_node_index(ni)) impact_string = safety.check_impact_string( response=response, shards=shards, # pyre-ignore target_state=mv.shard_target_state, ) return f"Safety Check Impact:\n\n{impact_string}" else: return ""
def shards_table(mv: MaintenanceView, cv: ClusterView, ni: int) -> str: headers = [ "SHARD INDEX", "CURRENT STATE", "TARGET STATE", "MAINTENANCE STATUS", "LAST UPDATED", ] tbl = [] for shard in mv.get_shards_by_node_index(ni): target_state = mv.shard_target_state shard_state = mv.get_shard_state(shard) cur_op_state = shard_state.current_operational_state current_state = colored( cur_op_state.name, _color_shard_op_state( cur_op_state, # pyre-ignore mv.shard_target_state, ), ) mnt_status = mv.get_shard_maintenance_status(shard) maintenance_status = colored( mnt_status.name, _color(mnt_status)) last_updated_at = mv.get_shard_last_updated_at( shard) if last_updated_at: last_updated = ( f"{last_updated_at} " f"({naturaltime(last_updated_at)})") else: last_updated = "-" tbl.append([ shard.shard_index, current_state, # pyre-ignore target_state.name, maintenance_status, last_updated, ]) return tabulate(tbl, headers=headers, tablefmt="plain")
def validate( self, maintenance_view: MaintenanceView, maintenance: MaintenanceDefinition, node_index_to_node_view: Dict[int, NodeView], ) -> None: self.assertEqual(maintenance_view.allow_passive_drains, maintenance.allow_passive_drains) self.assertListEqual( sorted(ni.node_index for ni in maintenance_view.affected_sequencer_node_ids), sorted(sn.node_index for sn in maintenance.sequencer_nodes), ) self.assertTupleEqual( maintenance_view.affected_sequencer_node_indexes, tuple(sorted(sn.node_index for sn in maintenance.sequencer_nodes)), ) self.assertListEqual( sorted(ni.node_index for ni in maintenance_view.affected_storage_node_ids), sorted({shard.node.node_index for shard in maintenance.shards}), ) self.assertTupleEqual( maintenance_view.affected_storage_node_indexes, tuple( sorted({shard.node.node_index for shard in maintenance.shards})), ) self.assertListEqual( sorted(ni.node_index for ni in maintenance_view.affected_node_ids), sorted({sn.node_index for sn in maintenance.sequencer_nodes}.union({ shard.node.node_index for shard in maintenance.shards })), ) self.assertTupleEqual( maintenance_view.affected_node_indexes, tuple( sorted({sn.node_index for sn in maintenance.sequencer_nodes}.union({ shard.node.node_index for shard in maintenance.shards }))), ) if len(maintenance.shards) == 0: self.assertIsNone(maintenance_view.shard_target_state) else: self.assertEqual(maintenance_view.shard_target_state, maintenance.shard_target_state) if len(maintenance.sequencer_nodes) == 0: self.assertIsNone(maintenance_view.sequencer_target_state) else: self.assertEqual( maintenance_view.sequencer_target_state, maintenance.sequencer_target_state, ) if maintenance.ttl_seconds == 0: self.assertIsNone(maintenance_view.ttl) else: assert maintenance_view.ttl is not None self.assertEqual( maintenance.ttl_seconds, # pyre-fixme[16]: Optional type has no attribute `total_seconds`. maintenance_view.ttl.total_seconds(), ) if maintenance.created_on is None: self.assertIsNone(maintenance_view.created_on) else: assert maintenance_view.created_on is not None self.assertAlmostEqual( # pyre-fixme[16]: Optional type has no attribute `timestamp`. maintenance_view.created_on.timestamp() * 1000, maintenance.created_on, 1, ) if maintenance.expires_on is None: self.assertIsNone(maintenance_view.expires_on) else: assert maintenance_view.expires_on is not None self.assertAlmostEqual( maintenance_view.expires_on.timestamp() * 1000, maintenance.expires_on, 1, ) assert maintenance_view.expires_in is not None self.assertAlmostEqual( maintenance_view.expires_in.total_seconds(), # pyre-fixme[16]: Optional type has no attribute `__sub__`. (maintenance_view.expires_on - datetime.now()).total_seconds(), 1, ) self.assertEqual(maintenance_view.affects_shards, len(maintenance.shards) > 0) self.assertEqual(maintenance_view.affects_sequencers, len(maintenance.sequencer_nodes) > 0) self.assertEqual(maintenance_view.num_shards_total, len(maintenance.shards)) self.assertEqual(maintenance_view.num_sequencers_total, len(maintenance.sequencer_nodes)) for shard in maintenance.shards: assert shard.node.node_index is not None self.assertEqual( # pyre-fixme[6]: Expected `int` for 1st param but got `Optional[int]`. node_index_to_node_view[shard.node.node_index ].shard_states[shard.shard_index], maintenance_view.get_shard_state(shard), ) for sn in maintenance.sequencer_nodes: assert sn.node_index is not None self.assertEqual( # pyre-fixme[6]: Expected `int` for 1st param but got `Optional[int]`. node_index_to_node_view[sn.node_index].sequencer_state, maintenance_view.get_sequencer_state(sn), ) self.assertSetEqual(set(maintenance.shards), set(maintenance_view.shards)) for node_index in { shard.node.node_index for shard in maintenance.shards }: assert node_index is not None shards = { shard for shard in maintenance.shards if shard.node.node_index == node_index } self.assertSetEqual( shards, set(maintenance_view.get_shards_by_node_index(node_index)))
def aggregated(mv: MaintenanceView, cv: ClusterView) -> str: headers = [ "NODE INDEX", "NODE NAME", "LOCATION", "TARGET STATE", "CURRENT STATE", "MAINTENANCE STATUS", "LAST UPDATED", ] tbl = [] for ni in mv.affected_storage_node_indexes: nv = cv.get_node_view(node_index=ni) node_index = ni node_name = nv.node_name location = nv.location target_state = ( # pyre-ignore f"{mv.shard_target_state.name}" f"({len(mv.get_shards_by_node_index(ni))})") # current_state chunks = [] for cur_op_state, num in sorted( Counter( (mv.get_shard_state( shard).current_operational_state for shard in mv.get_shards_by_node_index(ni) )).items(), key=lambda x: x[0].name, ): chunks.append( colored( f"{cur_op_state.name}({num})", _color_shard_op_state( cur_op_state, # pyre-ignore mv.shard_target_state, ), )) current_state = ",".join(chunks) # maintenance status mnt_statuses = [ mv.get_shard_maintenance_status(shard) for shard in mv.get_shards_by_node_index(ni) ] chunks = [] for mnt_status, num in sorted( Counter(mnt_statuses).items(), key=lambda x: x[0].name): chunks.append( colored(f"{mnt_status.name}({num})", _color(mnt_status))) maintenance_status = ",".join(chunks) last_updated_at_time = min(( ShardMaintenanceProgress.from_thrift( # pyre-ignore ss.maintenance).last_updated_at for ss in nv.shard_states if ss.maintenance)) last_updated_at = ( f"{last_updated_at_time} ({naturaltime(last_updated_at_time)})" ) tbl.append([ node_index, node_name, location, target_state, current_state, maintenance_status, last_updated_at, ]) return tabulate(tabular_data=tbl, headers=headers, tablefmt="plain")
def validate( self, maintenance_view: MaintenanceView, maintenance: MaintenanceDefinition, node_index_to_node_view: Dict[int, NodeView], ) -> None: self.assertEqual( maintenance_view.allow_passive_drains, maintenance.allow_passive_drains ) self.assertListEqual( sorted( # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got # `Generator[typing.Optional[int], None, None]`. ni.node_index for ni in maintenance_view.affected_sequencer_node_ids ), # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got # `Generator[typing.Optional[int], None, None]`. sorted(sn.node_index for sn in maintenance.sequencer_nodes), ) self.assertTupleEqual( maintenance_view.affected_sequencer_node_indexes, # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got # `Generator[typing.Optional[int], None, None]`. tuple(sorted(sn.node_index for sn in maintenance.sequencer_nodes)), ) self.assertListEqual( # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got # `Generator[typing.Optional[int], None, None]`. sorted(ni.node_index for ni in maintenance_view.affected_storage_node_ids), # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got `Set[typing.Optional[int]]`. sorted({shard.node.node_index for shard in maintenance.shards}), ) self.assertTupleEqual( maintenance_view.affected_storage_node_indexes, # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got `Set[typing.Optional[int]]`. tuple(sorted({shard.node.node_index for shard in maintenance.shards})), ) self.assertListEqual( # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got # `Generator[typing.Optional[int], None, None]`. sorted(ni.node_index for ni in maintenance_view.affected_node_ids), sorted( # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got # `Set[typing.Optional[int]]`. {sn.node_index for sn in maintenance.sequencer_nodes}.union( {shard.node.node_index for shard in maintenance.shards} ) ), ) self.assertTupleEqual( maintenance_view.affected_node_indexes, tuple( sorted( # pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to # _SupportsLessThan)]]` for 1st param but got # `Set[typing.Optional[int]]`. {sn.node_index for sn in maintenance.sequencer_nodes}.union( {shard.node.node_index for shard in maintenance.shards} ) ) ), ) if len(maintenance.shards) == 0: self.assertIsNone(maintenance_view.shard_target_state) else: self.assertEqual( maintenance_view.shard_target_state, maintenance.shard_target_state ) if len(maintenance.sequencer_nodes) == 0: self.assertIsNone(maintenance_view.sequencer_target_state) else: self.assertEqual( maintenance_view.sequencer_target_state, maintenance.sequencer_target_state, ) if maintenance.ttl_seconds == 0: self.assertIsNone(maintenance_view.ttl) else: assert maintenance_view.ttl is not None self.assertEqual( maintenance.ttl_seconds, # pyre-fixme[16]: Optional type has no attribute `total_seconds`. maintenance_view.ttl.total_seconds(), ) if maintenance.created_on is None: self.assertIsNone(maintenance_view.created_on) else: assert maintenance_view.created_on is not None self.assertAlmostEqual( # pyre-fixme[16]: Optional type has no attribute `timestamp`. maintenance_view.created_on.timestamp() * 1000, maintenance.created_on, 1, ) if maintenance.expires_on is None: self.assertIsNone(maintenance_view.expires_on) else: # pull these into local vars to appease pyre view_expires_on = maintenance_view.expires_on view_expires_in = maintenance_view.expires_in assert view_expires_on is not None self.assertAlmostEqual( view_expires_on.timestamp() * 1000, maintenance.expires_on, 1 ) assert view_expires_in is not None self.assertAlmostEqual( view_expires_in.total_seconds(), (view_expires_on - datetime.now()).total_seconds(), 1, ) self.assertEqual(maintenance_view.affects_shards, len(maintenance.shards) > 0) self.assertEqual( maintenance_view.affects_sequencers, len(maintenance.sequencer_nodes) > 0 ) self.assertEqual(maintenance_view.num_shards_total, len(maintenance.shards)) self.assertEqual( maintenance_view.num_sequencers_total, len(maintenance.sequencer_nodes) ) for shard in maintenance.shards: assert shard.node.node_index is not None self.assertEqual( node_index_to_node_view[shard.node.node_index].shard_states[ shard.shard_index ], maintenance_view.get_shard_state(shard), ) for sn in maintenance.sequencer_nodes: assert sn.node_index is not None self.assertEqual( node_index_to_node_view[sn.node_index].sequencer_state, maintenance_view.get_sequencer_state(sn), ) self.assertSetEqual(set(maintenance.shards), set(maintenance_view.shards)) for node_index in {shard.node.node_index for shard in maintenance.shards}: assert node_index is not None shards = { shard for shard in maintenance.shards if shard.node.node_index == node_index } self.assertSetEqual( shards, set(maintenance_view.get_shards_by_node_index(node_index)) )