async def test_get_shard_last_updated_at(self):
async with MockAdminAPI() as client:
cv = await get_cluster_view(client)
shard = ShardID(node=cv.get_node_view_by_node_index(0).node_id,
shard_index=1)
await apply_maintenance(
client=client,
shards=[shard],
shard_target_state=ShardOperationalState.DRAINED,
)
cv = await get_cluster_view(client)
mv = list(cv.get_all_maintenance_views())[0]
self.assertIsNone(mv.get_shard_last_updated_at(shard))
ts = datetime.now()
client._set_shard_maintenance_progress(
shard,
ShardMaintenanceProgress(
status=MaintenanceStatus.STARTED,
target_states=[ShardOperationalState.MAY_DISAPPEAR],
created_at=ts,
last_updated_at=datetime.now(),
associated_group_ids=["johnsnow"],
).to_thrift(),
)
cv = await get_cluster_view(client)
mv = list(cv.get_all_maintenance_views())[0]
self.assertTrue(
mv.get_shard_last_updated_at(shard) -
ts < timedelta(seconds=2))
def get_shard_last_updated_at(self, shard: ShardID) -> Optional[datetime]:
shard_state = self.get_shard_state(shard)
if shard_state.maintenance is not None:
return ShardMaintenanceProgress.from_thrift(
shard_state.maintenance).last_updated_at
else:
return None
def get_shard_last_updated_at(self, shard: ShardID) -> Optional[datetime]:
shard_state = self.get_shard_state(shard)
if shard_state.maintenance is not None:
return ShardMaintenanceProgress.from_thrift(
# pyre-fixme[6]: Expected `ShardMaintenanceProgress` for 1st param
# but got `Optional[ShardMaintenanceProgress]`.
shard_state.maintenance).last_updated_at
else:
return None
async def test_smoke(self):
created_at = datetime.now() - timedelta(hours=2)
last_updated_at = datetime.now() - timedelta(minutes=5)
maintenance_status = MaintenanceStatus.NOT_STARTED
target_states = {
ShardOperationalState.MAY_DISAPPEAR,
ShardOperationalState.DRAINED,
}
associated_group_ids = ["johnsnow"]
thrift_smp = ThriftShardMaintenanceProgress(
status=maintenance_status,
target_states=target_states,
created_at=int(created_at.timestamp() * 1000),
last_updated_at=int(last_updated_at.timestamp() * 1000),
associated_group_ids=associated_group_ids,
)
smp = ShardMaintenanceProgress(
status=maintenance_status,
target_states=target_states,
created_at=created_at,
last_updated_at=last_updated_at,
associated_group_ids=associated_group_ids,
)
self.assertEqual(thrift_smp, smp.to_thrift())
self.assertEqual(thrift_smp.status, smp.status)
self.assertEqual(thrift_smp.created_at // 1000,
int(smp.created_at.timestamp()))
self.assertEqual(thrift_smp.last_updated_at // 1000,
int(smp.last_updated_at.timestamp()))
from_thrift = ShardMaintenanceProgress.from_thrift(thrift_smp)
self.assertEqual(smp.status, from_thrift.status)
self.assertSetEqual(set(smp.target_states),
set(from_thrift.target_states))
self.assertEqual(smp.created_at, from_thrift.created_at)
self.assertEqual(smp.last_updated_at, from_thrift.last_updated_at)
self.assertSetEqual(set(smp.associated_group_ids),
set(from_thrift.associated_group_ids))
def aggregated(mv: MaintenanceView, cv: ClusterView) -> str:
headers = [
"NODE INDEX",
"NODE NAME",
"LOCATION",
"TARGET STATE",
"CURRENT STATE",
"MAINTENANCE STATUS",
"LAST UPDATED",
]
tbl = []
for ni in mv.affected_storage_node_indexes:
nv = cv.get_node_view(node_index=ni)
node_index = ni
node_name = nv.node_name
location = nv.location
target_state = (
# pyre-ignore
f"{mv.shard_target_state.name}"
f"({len(mv.get_shards_by_node_index(ni))})")
# current_state
chunks = []
for cur_op_state, num in sorted(
Counter(
(mv.get_shard_state(
shard).current_operational_state
for shard in mv.get_shards_by_node_index(ni)
)).items(),
key=lambda x: x[0].name,
):
chunks.append(
colored(
f"{cur_op_state.name}({num})",
_color_shard_op_state(
cur_op_state,
# pyre-ignore
mv.shard_target_state,
),
))
current_state = ",".join(chunks)
# maintenance status
mnt_statuses = [
mv.get_shard_maintenance_status(shard)
for shard in mv.get_shards_by_node_index(ni)
]
chunks = []
for mnt_status, num in sorted(
Counter(mnt_statuses).items(),
key=lambda x: x[0].name):
chunks.append(
colored(f"{mnt_status.name}({num})",
_color(mnt_status)))
maintenance_status = ",".join(chunks)
last_updated_at_time = min((
ShardMaintenanceProgress.from_thrift(
# pyre-ignore
ss.maintenance).last_updated_at
for ss in nv.shard_states if ss.maintenance))
last_updated_at = (
f"{last_updated_at_time} ({naturaltime(last_updated_at_time)})"
)
tbl.append([
node_index,
node_name,
location,
target_state,
current_state,
maintenance_status,
last_updated_at,
])
return tabulate(tabular_data=tbl,
headers=headers,
tablefmt="plain")
async def test_shard_maintenance_status(self):
## MAY_DISAPPEAR maintenance
async with MockAdminAPI() as client:
cv = await get_cluster_view(client)
shard = ShardID(node=cv.get_node_view_by_node_index(0).node_id,
shard_index=1)
await apply_maintenance(
client=client,
shards=[shard],
shard_target_state=ShardOperationalState.MAY_DISAPPEAR,
)
cv = await get_cluster_view(client)
# Just started
mv = list(cv.get_all_maintenance_views())[0]
self.assertEqual(mv.get_shard_maintenance_status(shard),
MaintenanceStatus.NOT_STARTED)
self.assertEqual(mv.num_shards_done, 0)
self.assertFalse(mv.are_all_shards_done)
self.assertFalse(mv.is_everything_done)
self.assertFalse(mv.is_blocked)
self.assertFalse(mv.is_completed)
self.assertTrue(mv.is_in_progress)
self.assertFalse(mv.is_internal)
self.assertEqual(mv.overall_status,
MaintenanceOverallStatus.IN_PROGRESS)
# In progress
client._set_shard_maintenance_progress(
shard,
ShardMaintenanceProgress(
status=MaintenanceStatus.STARTED,
target_states=[ShardOperationalState.MAY_DISAPPEAR],
created_at=datetime.now(),
last_updated_at=datetime.now(),
associated_group_ids=["johnsnow"],
).to_thrift(),
)
cv = await get_cluster_view(client)
mv = list(cv.get_all_maintenance_views())[0]
self.assertEqual(mv.get_shard_maintenance_status(shard),
MaintenanceStatus.STARTED)
self.assertEqual(mv.num_shards_done, 0)
self.assertFalse(mv.are_all_shards_done)
self.assertFalse(mv.is_everything_done)
self.assertFalse(mv.is_blocked)
self.assertFalse(mv.is_completed)
self.assertTrue(mv.is_in_progress)
self.assertFalse(mv.is_internal)
self.assertEqual(mv.overall_status,
MaintenanceOverallStatus.IN_PROGRESS)
# Blocked
client._set_shard_maintenance_progress(
shard,
ShardMaintenanceProgress(
status=MaintenanceStatus.BLOCKED_UNTIL_SAFE,
target_states=[ShardOperationalState.MAY_DISAPPEAR],
created_at=datetime.now(),
last_updated_at=datetime.now(),
associated_group_ids=["johnsnow"],
).to_thrift(),
)
cv = await get_cluster_view(client)
mv = list(cv.get_all_maintenance_views())[0]
self.assertTrue(mv.is_blocked)
self.assertFalse(mv.are_all_shards_done)
self.assertFalse(mv.is_everything_done)
self.assertTrue(mv.is_blocked)
self.assertFalse(mv.is_completed)
self.assertFalse(mv.is_in_progress)
self.assertEqual(mv.overall_status,
MaintenanceOverallStatus.BLOCKED)
# Done
for sos in {
ShardOperationalState.DRAINED,
ShardOperationalState.MAY_DISAPPEAR,
ShardOperationalState.MIGRATING_DATA,
ShardOperationalState.PROVISIONING,
}:
client._set_shard_current_operational_state(shard, sos)
cv = await get_cluster_view(client)
mv = list(cv.get_all_maintenance_views())[0]
self.assertEqual(mv.get_shard_maintenance_status(shard),
MaintenanceStatus.COMPLETED)
self.assertEqual(mv.num_shards_done, 1)
self.assertTrue(mv.are_all_shards_done)
self.assertTrue(mv.is_everything_done)
self.assertFalse(mv.is_blocked)
self.assertTrue(mv.is_completed)
self.assertFalse(mv.is_in_progress)
self.assertEqual(mv.overall_status,
MaintenanceOverallStatus.COMPLETED)
## DRAINED maintenance
async with MockAdminAPI() as client:
cv = await get_cluster_view(client)
shard = ShardID(node=cv.get_node_view_by_node_index(0).node_id,
shard_index=1)
await apply_maintenance(
client=client,
shards=[shard],
shard_target_state=ShardOperationalState.DRAINED,
)
cv = await get_cluster_view(client)
# Just started
mv = list(cv.get_all_maintenance_views())[0]
self.assertEqual(mv.get_shard_maintenance_status(shard),
MaintenanceStatus.NOT_STARTED)
self.assertEqual(mv.num_shards_done, 0)
self.assertFalse(mv.are_all_shards_done)
# May disappear
client._set_shard_current_operational_state(
shard, ShardOperationalState.MAY_DISAPPEAR)
cv = await get_cluster_view(client)
mv = list(cv.get_all_maintenance_views())[0]
self.assertEqual(mv.get_shard_maintenance_status(shard),
MaintenanceStatus.NOT_STARTED)
self.assertEqual(mv.num_shards_done, 0)
self.assertFalse(mv.are_all_shards_done)
# Done
client._set_shard_current_operational_state(
shard, ShardOperationalState.DRAINED)
cv = await get_cluster_view(client)
mv = list(cv.get_all_maintenance_views())[0]
self.assertEqual(mv.get_shard_maintenance_status(shard),
MaintenanceStatus.COMPLETED)
self.assertEqual(mv.num_shards_done, 1)
self.assertTrue(mv.are_all_shards_done)