def sequencers(mv: MaintenanceView, cv: ClusterView) -> Optional[str]:
if not mv.affects_sequencers:
return None
headers = [
"NODE INDEX",
"NODE NAME",
"LOCATION",
"TARGET STATE",
"CURRENT STATE",
"MAINTENANCE STATUS",
"LAST UPDATED",
]
tbl = []
for n in mv.sequencer_nodes:
nv = cv.get_node_view(node_index=n.node_index)
node_index = nv.node_index
node_name = nv.node_name
location = nv.location
# pyre-ignore
target_state = mv.sequencer_target_state.name
mnt_status = mv.get_sequencer_maintenance_status(n)
current_state = colored(
# pyre-ignore
nv.sequencer_state.state.name,
_color(mnt_status),
)
maintenance_status = colored(mnt_status.name,
_color(mnt_status))
last_updated_at = mv.get_sequencer_last_updated_at(n)
if last_updated_at:
last_updated = f"{last_updated_at} ({naturaltime(last_updated_at)})"
else:
last_updated = "-"
tbl.append([
node_index,
node_name,
location,
target_state,
current_state,
maintenance_status,
last_updated,
])
return "Sequencer Maintenances:\n{}".format(
indent(tabulate(tbl, headers=headers, tablefmt="plain"),
prefix=" "))
def shards_table(mv: MaintenanceView, cv: ClusterView,
ni: int) -> str:
headers = [
"SHARD INDEX",
"CURRENT STATE",
"TARGET STATE",
"MAINTENANCE STATUS",
"LAST UPDATED",
]
tbl = []
for shard in mv.get_shards_by_node_index(ni):
target_state = mv.shard_target_state
shard_state = mv.get_shard_state(shard)
cur_op_state = shard_state.current_operational_state
current_state = colored(
cur_op_state.name,
_color_shard_op_state(
cur_op_state,
# pyre-ignore
mv.shard_target_state,
),
)
mnt_status = mv.get_shard_maintenance_status(shard)
maintenance_status = colored(
mnt_status.name, _color(mnt_status))
last_updated_at = mv.get_shard_last_updated_at(
shard)
if last_updated_at:
last_updated = (
f"{last_updated_at} "
f"({naturaltime(last_updated_at)})")
else:
last_updated = "-"
tbl.append([
shard.shard_index,
current_state,
# pyre-ignore
target_state.name,
maintenance_status,
last_updated,
])
return tabulate(tbl, headers=headers, tablefmt="plain")
def _maintenance_id_to_maintenance_view(self) -> Dict[str, MaintenanceView]:
if self._maintenance_id_to_maintenance_view_dict is None:
self._maintenance_id_to_maintenance_view_dict = {
mnt_id: MaintenanceView(
maintenance=self._maintenance_id_to_maintenance[mnt_id],
node_index_to_node_view={
ni: self._node_index_to_node_view[ni]
for ni in self._maintenance_id_to_node_indexes[mnt_id]
},
)
for mnt_id in self._maintenance_ids
}
return self._maintenance_id_to_maintenance_view_dict
def _maintenance_id_to_maintenance_view(self) -> Dict[str, MaintenanceView]:
if self._maintenance_id_to_maintenance_view_dict is None:
self._maintenance_id_to_maintenance_view_dict = {
mnt_id: MaintenanceView(
maintenance=self._maintenance_id_to_maintenance[mnt_id],
node_index_to_node_view={
ni: self._node_index_to_node_view[ni]
for ni in self._maintenance_id_to_node_indexes[mnt_id]
},
)
for mnt_id in self._maintenance_ids
}
# pyre-fixme[7]: Expected `Dict[str, MaintenanceView]` but got
# `Optional[Dict[str, MaintenanceView]]`.
return self._maintenance_id_to_maintenance_view_dict
def impact(mv: MaintenanceView,
show_safety_check_results: bool) -> str:
if (mv.overall_status == MaintenanceOverallStatus.BLOCKED
and show_safety_check_results):
response = mv.last_check_impact_result
shards = []
for ni in mv.affected_node_indexes:
shards.extend(mv.get_shards_by_node_index(ni))
impact_string = safety.check_impact_string(
response=response,
shards=shards,
# pyre-ignore
target_state=mv.shard_target_state,
)
return f"Safety Check Impact:\n\n{impact_string}"
else:
return ""
async def test_no_timestamps(self):
async with MockAdminAPI() as client:
cv = await get_cluster_view(client)
await apply_maintenance(
client=client,
sequencer_nodes=[cv.get_node_view_by_node_index(0).node_id],
ttl=timedelta(days=2),
)
cv = await get_cluster_view(client)
mnt = list(cv.get_all_maintenances())[0]
mnt = mnt(created_on=None, expires_on=None)
mv = MaintenanceView(
maintenance=mnt,
node_index_to_node_view={0: cv.get_node_view(node_index=0)})
self.assertIsNone(mv.created_on)
self.assertIsNone(mv.expires_on)
self.assertIsNone(mv.expires_in)
def aggregated(mv: MaintenanceView, cv: ClusterView) -> str:
headers = [
"NODE INDEX",
"NODE NAME",
"LOCATION",
"TARGET STATE",
"CURRENT STATE",
"MAINTENANCE STATUS",
"LAST UPDATED",
]
tbl = []
for ni in mv.affected_storage_node_indexes:
nv = cv.get_node_view(node_index=ni)
node_index = ni
node_name = nv.node_name
location = nv.location
target_state = (
# pyre-ignore
f"{mv.shard_target_state.name}"
f"({len(mv.get_shards_by_node_index(ni))})")
# current_state
chunks = []
for cur_op_state, num in sorted(
Counter(
(mv.get_shard_state(
shard).current_operational_state
for shard in mv.get_shards_by_node_index(ni)
)).items(),
key=lambda x: x[0].name,
):
chunks.append(
colored(
f"{cur_op_state.name}({num})",
_color_shard_op_state(
cur_op_state,
# pyre-ignore
mv.shard_target_state,
),
))
current_state = ",".join(chunks)
# maintenance status
mnt_statuses = [
mv.get_shard_maintenance_status(shard)
for shard in mv.get_shards_by_node_index(ni)
]
chunks = []
for mnt_status, num in sorted(
Counter(mnt_statuses).items(),
key=lambda x: x[0].name):
chunks.append(
colored(f"{mnt_status.name}({num})",
_color(mnt_status)))
maintenance_status = ",".join(chunks)
last_updated_at_time = min((
ShardMaintenanceProgress.from_thrift(
# pyre-ignore
ss.maintenance).last_updated_at
for ss in nv.shard_states if ss.maintenance))
last_updated_at = (
f"{last_updated_at_time} ({naturaltime(last_updated_at_time)})"
)
tbl.append([
node_index,
node_name,
location,
target_state,
current_state,
maintenance_status,
last_updated_at,
])
return tabulate(tabular_data=tbl,
headers=headers,
tablefmt="plain")
def validate(
self,
maintenance_view: MaintenanceView,
maintenance: MaintenanceDefinition,
node_index_to_node_view: Dict[int, NodeView],
) -> None:
self.assertEqual(maintenance_view.allow_passive_drains,
maintenance.allow_passive_drains)
self.assertListEqual(
sorted(ni.node_index
for ni in maintenance_view.affected_sequencer_node_ids),
sorted(sn.node_index for sn in maintenance.sequencer_nodes),
)
self.assertTupleEqual(
maintenance_view.affected_sequencer_node_indexes,
tuple(sorted(sn.node_index for sn in maintenance.sequencer_nodes)),
)
self.assertListEqual(
sorted(ni.node_index
for ni in maintenance_view.affected_storage_node_ids),
sorted({shard.node.node_index
for shard in maintenance.shards}),
)
self.assertTupleEqual(
maintenance_view.affected_storage_node_indexes,
tuple(
sorted({shard.node.node_index
for shard in maintenance.shards})),
)
self.assertListEqual(
sorted(ni.node_index for ni in maintenance_view.affected_node_ids),
sorted({sn.node_index
for sn in maintenance.sequencer_nodes}.union({
shard.node.node_index
for shard in maintenance.shards
})),
)
self.assertTupleEqual(
maintenance_view.affected_node_indexes,
tuple(
sorted({sn.node_index
for sn in maintenance.sequencer_nodes}.union({
shard.node.node_index
for shard in maintenance.shards
}))),
)
if len(maintenance.shards) == 0:
self.assertIsNone(maintenance_view.shard_target_state)
else:
self.assertEqual(maintenance_view.shard_target_state,
maintenance.shard_target_state)
if len(maintenance.sequencer_nodes) == 0:
self.assertIsNone(maintenance_view.sequencer_target_state)
else:
self.assertEqual(
maintenance_view.sequencer_target_state,
maintenance.sequencer_target_state,
)
if maintenance.ttl_seconds == 0:
self.assertIsNone(maintenance_view.ttl)
else:
assert maintenance_view.ttl is not None
self.assertEqual(
maintenance.ttl_seconds,
# pyre-fixme[16]: Optional type has no attribute `total_seconds`.
maintenance_view.ttl.total_seconds(),
)
if maintenance.created_on is None:
self.assertIsNone(maintenance_view.created_on)
else:
assert maintenance_view.created_on is not None
self.assertAlmostEqual(
# pyre-fixme[16]: Optional type has no attribute `timestamp`.
maintenance_view.created_on.timestamp() * 1000,
maintenance.created_on,
1,
)
if maintenance.expires_on is None:
self.assertIsNone(maintenance_view.expires_on)
else:
assert maintenance_view.expires_on is not None
self.assertAlmostEqual(
maintenance_view.expires_on.timestamp() * 1000,
maintenance.expires_on,
1,
)
assert maintenance_view.expires_in is not None
self.assertAlmostEqual(
maintenance_view.expires_in.total_seconds(),
# pyre-fixme[16]: Optional type has no attribute `__sub__`.
(maintenance_view.expires_on - datetime.now()).total_seconds(),
1,
)
self.assertEqual(maintenance_view.affects_shards,
len(maintenance.shards) > 0)
self.assertEqual(maintenance_view.affects_sequencers,
len(maintenance.sequencer_nodes) > 0)
self.assertEqual(maintenance_view.num_shards_total,
len(maintenance.shards))
self.assertEqual(maintenance_view.num_sequencers_total,
len(maintenance.sequencer_nodes))
for shard in maintenance.shards:
assert shard.node.node_index is not None
self.assertEqual(
# pyre-fixme[6]: Expected `int` for 1st param but got `Optional[int]`.
node_index_to_node_view[shard.node.node_index
].shard_states[shard.shard_index],
maintenance_view.get_shard_state(shard),
)
for sn in maintenance.sequencer_nodes:
assert sn.node_index is not None
self.assertEqual(
# pyre-fixme[6]: Expected `int` for 1st param but got `Optional[int]`.
node_index_to_node_view[sn.node_index].sequencer_state,
maintenance_view.get_sequencer_state(sn),
)
self.assertSetEqual(set(maintenance.shards),
set(maintenance_view.shards))
for node_index in {
shard.node.node_index
for shard in maintenance.shards
}:
assert node_index is not None
shards = {
shard
for shard in maintenance.shards
if shard.node.node_index == node_index
}
self.assertSetEqual(
shards,
set(maintenance_view.get_shards_by_node_index(node_index)))
def _get_shard_operational_state(mv: MaintenanceView,
shard: ShardID) -> ShardOperationalState:
shard_state = mv.get_shard_state(shard)
if shard_state is not None:
return shard_state.current_operational_state
return ShardOperationalState.UNKNOWN
def validate(
self,
maintenance_view: MaintenanceView,
maintenance: MaintenanceDefinition,
node_index_to_node_view: Dict[int, NodeView],
) -> None:
self.assertEqual(
maintenance_view.allow_passive_drains, maintenance.allow_passive_drains
)
self.assertListEqual(
sorted(
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Generator[typing.Optional[int], None, None]`.
ni.node_index
for ni in maintenance_view.affected_sequencer_node_ids
),
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Generator[typing.Optional[int], None, None]`.
sorted(sn.node_index for sn in maintenance.sequencer_nodes),
)
self.assertTupleEqual(
maintenance_view.affected_sequencer_node_indexes,
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Generator[typing.Optional[int], None, None]`.
tuple(sorted(sn.node_index for sn in maintenance.sequencer_nodes)),
)
self.assertListEqual(
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Generator[typing.Optional[int], None, None]`.
sorted(ni.node_index for ni in maintenance_view.affected_storage_node_ids),
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got `Set[typing.Optional[int]]`.
sorted({shard.node.node_index for shard in maintenance.shards}),
)
self.assertTupleEqual(
maintenance_view.affected_storage_node_indexes,
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got `Set[typing.Optional[int]]`.
tuple(sorted({shard.node.node_index for shard in maintenance.shards})),
)
self.assertListEqual(
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Generator[typing.Optional[int], None, None]`.
sorted(ni.node_index for ni in maintenance_view.affected_node_ids),
sorted(
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Set[typing.Optional[int]]`.
{sn.node_index for sn in maintenance.sequencer_nodes}.union(
{shard.node.node_index for shard in maintenance.shards}
)
),
)
self.assertTupleEqual(
maintenance_view.affected_node_indexes,
tuple(
sorted(
# pyre-fixme[6]: Expected `Iterable[Variable[_LT (bound to
# _SupportsLessThan)]]` for 1st param but got
# `Set[typing.Optional[int]]`.
{sn.node_index for sn in maintenance.sequencer_nodes}.union(
{shard.node.node_index for shard in maintenance.shards}
)
)
),
)
if len(maintenance.shards) == 0:
self.assertIsNone(maintenance_view.shard_target_state)
else:
self.assertEqual(
maintenance_view.shard_target_state, maintenance.shard_target_state
)
if len(maintenance.sequencer_nodes) == 0:
self.assertIsNone(maintenance_view.sequencer_target_state)
else:
self.assertEqual(
maintenance_view.sequencer_target_state,
maintenance.sequencer_target_state,
)
if maintenance.ttl_seconds == 0:
self.assertIsNone(maintenance_view.ttl)
else:
assert maintenance_view.ttl is not None
self.assertEqual(
maintenance.ttl_seconds,
# pyre-fixme[16]: Optional type has no attribute `total_seconds`.
maintenance_view.ttl.total_seconds(),
)
if maintenance.created_on is None:
self.assertIsNone(maintenance_view.created_on)
else:
assert maintenance_view.created_on is not None
self.assertAlmostEqual(
# pyre-fixme[16]: Optional type has no attribute `timestamp`.
maintenance_view.created_on.timestamp() * 1000,
maintenance.created_on,
1,
)
if maintenance.expires_on is None:
self.assertIsNone(maintenance_view.expires_on)
else:
# pull these into local vars to appease pyre
view_expires_on = maintenance_view.expires_on
view_expires_in = maintenance_view.expires_in
assert view_expires_on is not None
self.assertAlmostEqual(
view_expires_on.timestamp() * 1000, maintenance.expires_on, 1
)
assert view_expires_in is not None
self.assertAlmostEqual(
view_expires_in.total_seconds(),
(view_expires_on - datetime.now()).total_seconds(),
1,
)
self.assertEqual(maintenance_view.affects_shards, len(maintenance.shards) > 0)
self.assertEqual(
maintenance_view.affects_sequencers, len(maintenance.sequencer_nodes) > 0
)
self.assertEqual(maintenance_view.num_shards_total, len(maintenance.shards))
self.assertEqual(
maintenance_view.num_sequencers_total, len(maintenance.sequencer_nodes)
)
for shard in maintenance.shards:
assert shard.node.node_index is not None
self.assertEqual(
node_index_to_node_view[shard.node.node_index].shard_states[
shard.shard_index
],
maintenance_view.get_shard_state(shard),
)
for sn in maintenance.sequencer_nodes:
assert sn.node_index is not None
self.assertEqual(
node_index_to_node_view[sn.node_index].sequencer_state,
maintenance_view.get_sequencer_state(sn),
)
self.assertSetEqual(set(maintenance.shards), set(maintenance_view.shards))
for node_index in {shard.node.node_index for shard in maintenance.shards}:
assert node_index is not None
shards = {
shard
for shard in maintenance.shards
if shard.node.node_index == node_index
}
self.assertSetEqual(
shards, set(maintenance_view.get_shards_by_node_index(node_index))
)