async def test_get_shard_last_updated_at(self): async with MockAdminAPI() as client: cv = await get_cluster_view(client) shard = ShardID(node=cv.get_node_view_by_node_index(0).node_id, shard_index=1) await apply_maintenance( client=client, shards=[shard], shard_target_state=ShardOperationalState.DRAINED, ) cv = await get_cluster_view(client) mv = list(cv.get_all_maintenance_views())[0] self.assertIsNone(mv.get_shard_last_updated_at(shard)) ts = datetime.now() client._set_shard_maintenance_progress( shard, ShardMaintenanceProgress( status=MaintenanceStatus.STARTED, target_states=[ShardOperationalState.MAY_DISAPPEAR], created_at=ts, last_updated_at=datetime.now(), associated_group_ids=["johnsnow"], ).to_thrift(), ) cv = await get_cluster_view(client) mv = list(cv.get_all_maintenance_views())[0] self.assertTrue( mv.get_shard_last_updated_at(shard) - ts < timedelta(seconds=2))
def get_shard_last_updated_at(self, shard: ShardID) -> Optional[datetime]: shard_state = self.get_shard_state(shard) if shard_state.maintenance is not None: return ShardMaintenanceProgress.from_thrift( shard_state.maintenance).last_updated_at else: return None
def get_shard_last_updated_at(self, shard: ShardID) -> Optional[datetime]: shard_state = self.get_shard_state(shard) if shard_state.maintenance is not None: return ShardMaintenanceProgress.from_thrift( # pyre-fixme[6]: Expected `ShardMaintenanceProgress` for 1st param # but got `Optional[ShardMaintenanceProgress]`. shard_state.maintenance).last_updated_at else: return None
async def test_smoke(self): created_at = datetime.now() - timedelta(hours=2) last_updated_at = datetime.now() - timedelta(minutes=5) maintenance_status = MaintenanceStatus.NOT_STARTED target_states = { ShardOperationalState.MAY_DISAPPEAR, ShardOperationalState.DRAINED, } associated_group_ids = ["johnsnow"] thrift_smp = ThriftShardMaintenanceProgress( status=maintenance_status, target_states=target_states, created_at=int(created_at.timestamp() * 1000), last_updated_at=int(last_updated_at.timestamp() * 1000), associated_group_ids=associated_group_ids, ) smp = ShardMaintenanceProgress( status=maintenance_status, target_states=target_states, created_at=created_at, last_updated_at=last_updated_at, associated_group_ids=associated_group_ids, ) self.assertEqual(thrift_smp, smp.to_thrift()) self.assertEqual(thrift_smp.status, smp.status) self.assertEqual(thrift_smp.created_at // 1000, int(smp.created_at.timestamp())) self.assertEqual(thrift_smp.last_updated_at // 1000, int(smp.last_updated_at.timestamp())) from_thrift = ShardMaintenanceProgress.from_thrift(thrift_smp) self.assertEqual(smp.status, from_thrift.status) self.assertSetEqual(set(smp.target_states), set(from_thrift.target_states)) self.assertEqual(smp.created_at, from_thrift.created_at) self.assertEqual(smp.last_updated_at, from_thrift.last_updated_at) self.assertSetEqual(set(smp.associated_group_ids), set(from_thrift.associated_group_ids))
def aggregated(mv: MaintenanceView, cv: ClusterView) -> str: headers = [ "NODE INDEX", "NODE NAME", "LOCATION", "TARGET STATE", "CURRENT STATE", "MAINTENANCE STATUS", "LAST UPDATED", ] tbl = [] for ni in mv.affected_storage_node_indexes: nv = cv.get_node_view(node_index=ni) node_index = ni node_name = nv.node_name location = nv.location target_state = ( # pyre-ignore f"{mv.shard_target_state.name}" f"({len(mv.get_shards_by_node_index(ni))})") # current_state chunks = [] for cur_op_state, num in sorted( Counter( (mv.get_shard_state( shard).current_operational_state for shard in mv.get_shards_by_node_index(ni) )).items(), key=lambda x: x[0].name, ): chunks.append( colored( f"{cur_op_state.name}({num})", _color_shard_op_state( cur_op_state, # pyre-ignore mv.shard_target_state, ), )) current_state = ",".join(chunks) # maintenance status mnt_statuses = [ mv.get_shard_maintenance_status(shard) for shard in mv.get_shards_by_node_index(ni) ] chunks = [] for mnt_status, num in sorted( Counter(mnt_statuses).items(), key=lambda x: x[0].name): chunks.append( colored(f"{mnt_status.name}({num})", _color(mnt_status))) maintenance_status = ",".join(chunks) last_updated_at_time = min(( ShardMaintenanceProgress.from_thrift( # pyre-ignore ss.maintenance).last_updated_at for ss in nv.shard_states if ss.maintenance)) last_updated_at = ( f"{last_updated_at_time} ({naturaltime(last_updated_at_time)})" ) tbl.append([ node_index, node_name, location, target_state, current_state, maintenance_status, last_updated_at, ]) return tabulate(tabular_data=tbl, headers=headers, tablefmt="plain")
async def test_shard_maintenance_status(self): ## MAY_DISAPPEAR maintenance async with MockAdminAPI() as client: cv = await get_cluster_view(client) shard = ShardID(node=cv.get_node_view_by_node_index(0).node_id, shard_index=1) await apply_maintenance( client=client, shards=[shard], shard_target_state=ShardOperationalState.MAY_DISAPPEAR, ) cv = await get_cluster_view(client) # Just started mv = list(cv.get_all_maintenance_views())[0] self.assertEqual(mv.get_shard_maintenance_status(shard), MaintenanceStatus.NOT_STARTED) self.assertEqual(mv.num_shards_done, 0) self.assertFalse(mv.are_all_shards_done) self.assertFalse(mv.is_everything_done) self.assertFalse(mv.is_blocked) self.assertFalse(mv.is_completed) self.assertTrue(mv.is_in_progress) self.assertFalse(mv.is_internal) self.assertEqual(mv.overall_status, MaintenanceOverallStatus.IN_PROGRESS) # In progress client._set_shard_maintenance_progress( shard, ShardMaintenanceProgress( status=MaintenanceStatus.STARTED, target_states=[ShardOperationalState.MAY_DISAPPEAR], created_at=datetime.now(), last_updated_at=datetime.now(), associated_group_ids=["johnsnow"], ).to_thrift(), ) cv = await get_cluster_view(client) mv = list(cv.get_all_maintenance_views())[0] self.assertEqual(mv.get_shard_maintenance_status(shard), MaintenanceStatus.STARTED) self.assertEqual(mv.num_shards_done, 0) self.assertFalse(mv.are_all_shards_done) self.assertFalse(mv.is_everything_done) self.assertFalse(mv.is_blocked) self.assertFalse(mv.is_completed) self.assertTrue(mv.is_in_progress) self.assertFalse(mv.is_internal) self.assertEqual(mv.overall_status, MaintenanceOverallStatus.IN_PROGRESS) # Blocked client._set_shard_maintenance_progress( shard, ShardMaintenanceProgress( status=MaintenanceStatus.BLOCKED_UNTIL_SAFE, target_states=[ShardOperationalState.MAY_DISAPPEAR], created_at=datetime.now(), last_updated_at=datetime.now(), associated_group_ids=["johnsnow"], ).to_thrift(), ) cv = await get_cluster_view(client) mv = list(cv.get_all_maintenance_views())[0] self.assertTrue(mv.is_blocked) self.assertFalse(mv.are_all_shards_done) self.assertFalse(mv.is_everything_done) self.assertTrue(mv.is_blocked) self.assertFalse(mv.is_completed) self.assertFalse(mv.is_in_progress) self.assertEqual(mv.overall_status, MaintenanceOverallStatus.BLOCKED) # Done for sos in { ShardOperationalState.DRAINED, ShardOperationalState.MAY_DISAPPEAR, ShardOperationalState.MIGRATING_DATA, ShardOperationalState.PROVISIONING, }: client._set_shard_current_operational_state(shard, sos) cv = await get_cluster_view(client) mv = list(cv.get_all_maintenance_views())[0] self.assertEqual(mv.get_shard_maintenance_status(shard), MaintenanceStatus.COMPLETED) self.assertEqual(mv.num_shards_done, 1) self.assertTrue(mv.are_all_shards_done) self.assertTrue(mv.is_everything_done) self.assertFalse(mv.is_blocked) self.assertTrue(mv.is_completed) self.assertFalse(mv.is_in_progress) self.assertEqual(mv.overall_status, MaintenanceOverallStatus.COMPLETED) ## DRAINED maintenance async with MockAdminAPI() as client: cv = await get_cluster_view(client) shard = ShardID(node=cv.get_node_view_by_node_index(0).node_id, shard_index=1) await apply_maintenance( client=client, shards=[shard], shard_target_state=ShardOperationalState.DRAINED, ) cv = await get_cluster_view(client) # Just started mv = list(cv.get_all_maintenance_views())[0] self.assertEqual(mv.get_shard_maintenance_status(shard), MaintenanceStatus.NOT_STARTED) self.assertEqual(mv.num_shards_done, 0) self.assertFalse(mv.are_all_shards_done) # May disappear client._set_shard_current_operational_state( shard, ShardOperationalState.MAY_DISAPPEAR) cv = await get_cluster_view(client) mv = list(cv.get_all_maintenance_views())[0] self.assertEqual(mv.get_shard_maintenance_status(shard), MaintenanceStatus.NOT_STARTED) self.assertEqual(mv.num_shards_done, 0) self.assertFalse(mv.are_all_shards_done) # Done client._set_shard_current_operational_state( shard, ShardOperationalState.DRAINED) cv = await get_cluster_view(client) mv = list(cv.get_all_maintenance_views())[0] self.assertEqual(mv.get_shard_maintenance_status(shard), MaintenanceStatus.COMPLETED) self.assertEqual(mv.num_shards_done, 1) self.assertTrue(mv.are_all_shards_done)