def test_subsetting_basic(self) -> None: with self.subTest('00_create_health_check'): self.td.create_health_check() with self.subTest('01_create_backend_services'): self.td.create_backend_service(subset_size=_SUBSET_SIZE) with self.subTest('02_create_url_map'): self.td.create_url_map(self.server_xds_host, self.server_xds_port) with self.subTest('03_create_target_proxy'): self.td.create_target_proxy() with self.subTest('04_create_forwarding_rule'): self.td.create_forwarding_rule(self.server_xds_port) test_servers: List[_XdsTestServer] with self.subTest('05_start_test_servers'): test_servers = self.startTestServers(replica_count=_NUM_BACKENDS) with self.subTest('06_add_server_backends_to_backend_services'): self.setupServerBackends() rpc_distribution = collections.defaultdict(int) with self.subTest('07_start_test_client'): for i in range(_NUM_CLIENTS): # Clean created client pods if there is any self.client_runner.cleanup(force=True) # Create a test client test_client: _XdsTestClient = self.startTestClient( test_servers[0]) # Validate the number of received endpoints config = test_client.csds.fetch_client_status( log_level=logging.INFO) self.assertIsNotNone(config) json_config = json_format.MessageToDict(config) parsed = xds_url_map_testcase.DumpedXdsConfig(json_config) logging.info('Client %d received endpoints (len=%s): %s', i, len(parsed.endpoints), parsed.endpoints) self.assertLen(parsed.endpoints, _SUBSET_SIZE) # Record RPC stats lb_stats = self.getClientRpcStats(test_client, _NUM_BACKENDS * 25) for key, value in lb_stats.rpcs_by_peer.items(): rpc_distribution[key] += value with self.subTest('08_log_rpc_distribution'): server_entries = sorted(rpc_distribution.items(), key=lambda x: -x[1]) # Validate if clients are receiving different sets of backends (3 # client received a total of 4 unique backends == FAIL, a total of 5 # unique backends == PASS) self.assertGreater(len(server_entries), _SUBSET_SIZE) logging.info('RPC distribution (len=%s): %s', len(server_entries), server_entries) peak = server_entries[0][1] mean = sum(map(lambda x: x[1], server_entries)) / len(server_entries) logging.info('Peak=%d Mean=%.1f Peak-to-Mean-Ratio=%.2f', peak, mean, peak / mean)
def assertRouteConfigUpdateTrafficHandoff( self, test_client: XdsTestClient, previous_route_config_version: str, retry_wait_second: int, timeout_second: int): retryer = retryers.constant_retryer( wait_fixed=datetime.timedelta(seconds=retry_wait_second), timeout=datetime.timedelta(seconds=timeout_second), retry_on_exceptions=(TdPropagationRetryableError, ), logger=logger, log_level=logging.INFO) try: for attempt in retryer: with attempt: self.assertSuccessfulRpcs(test_client) raw_config = test_client.csds.fetch_client_status( log_level=logging.INFO) dumped_config = xds_url_map_testcase.DumpedXdsConfig( json_format.MessageToDict(raw_config)) route_config_version = dumped_config.rds_version if previous_route_config_version == route_config_version: logger.info( 'Routing config not propagated yet. Retrying.') raise TdPropagationRetryableError( "CSDS not get updated routing config corresponding" " to the second set of url maps") else: self.assertSuccessfulRpcs(test_client) logger.info( '[SUCCESS] Confirmed successful RPC with the updated routing config, version=%s', route_config_version) except retryers.RetryError as retry_error: logger.info( 'Retry exhausted. TD routing config propagation failed after timeout %ds. Last seen client config dump: %s', timeout_second, dumped_config) raise retry_error
def test_affinity(self) -> None: # pylint: disable=too-many-statements with self.subTest('00_create_health_check'): self.td.create_health_check() with self.subTest('01_create_backend_services'): self.td.create_backend_service( affinity_header=_TEST_AFFINITY_METADATA_KEY) with self.subTest('02_create_url_map'): self.td.create_url_map(self.server_xds_host, self.server_xds_port) with self.subTest('03_create_target_proxy'): self.td.create_target_proxy() with self.subTest('04_create_forwarding_rule'): self.td.create_forwarding_rule(self.server_xds_port) test_servers: List[_XdsTestServer] with self.subTest('05_start_test_servers'): test_servers = self.startTestServers(replica_count=_REPLICA_COUNT) with self.subTest('06_add_server_backends_to_backend_services'): self.setupServerBackends() test_client: _XdsTestClient with self.subTest('07_start_test_client'): test_client = self.startTestClient(test_servers[0], rpc='EmptyCall', metadata='EmptyCall:%s:123' % _TEST_AFFINITY_METADATA_KEY) # Validate the number of received endpoints and affinity configs. config = test_client.csds.fetch_client_status( log_level=logging.INFO) self.assertIsNotNone(config) json_config = json_format.MessageToDict(config) parsed = xds_url_map_testcase.DumpedXdsConfig(json_config) logging.info('Client received CSDS response: %s', parsed) self.assertLen(parsed.endpoints, _REPLICA_COUNT) self.assertEqual( parsed.rds['virtualHosts'][0]['routes'][0]['route'] ['hashPolicy'][0]['header']['headerName'], _TEST_AFFINITY_METADATA_KEY) self.assertEqual(parsed.cds[0]['lbPolicy'], 'RING_HASH') with self.subTest('08_test_client_xds_config_exists'): self.assertXdsConfigExists(test_client) with self.subTest('09_test_server_received_rpcs_from_test_client'): self.assertSuccessfulRpcs(test_client) with self.subTest('10_first_100_affinity_rpcs_pick_same_backend'): rpc_stats = self.getClientRpcStats(test_client, _RPC_COUNT) json_lb_stats = json_format.MessageToDict(rpc_stats) rpc_distribution = xds_url_map_testcase.RpcDistributionStats( json_lb_stats) self.assertEqual(1, rpc_distribution.num_peers) self.assertLen( test_client.find_subchannels_with_state( _ChannelzChannelState.READY), 1, ) self.assertLen( test_client.find_subchannels_with_state( _ChannelzChannelState.IDLE), 2, ) # Remember the backend inuse, and turn it down later. first_backend_inuse = list( rpc_distribution.raw['rpcsByPeer'].keys())[0] with self.subTest('11_turn_down_server_in_use'): for server in test_servers: if server.hostname == first_backend_inuse: server.set_not_serving() with self.subTest('12_wait_for_unhealth_status_propagation'): deadline = time.time() + _TD_PROPAGATE_TIMEOUT parsed = None try: while time.time() < deadline: config = test_client.csds.fetch_client_status( log_level=logging.INFO) self.assertIsNotNone(config) json_config = json_format.MessageToDict(config) parsed = xds_url_map_testcase.DumpedXdsConfig(json_config) if len(parsed.endpoints) == _REPLICA_COUNT - 1: break logging.info( 'CSDS got unexpected endpoints, will retry after %d seconds', _TD_PROPAGATE_CHECK_INTERVAL_SEC) time.sleep(_TD_PROPAGATE_CHECK_INTERVAL_SEC) else: self.fail( 'unhealthy status did not propagate after 600 seconds') finally: logging.info('Client received CSDS response: %s', parsed) with self.subTest('12_next_100_affinity_rpcs_pick_different_backend'): rpc_stats = self.getClientRpcStats(test_client, _RPC_COUNT) json_lb_stats = json_format.MessageToDict(rpc_stats) rpc_distribution = xds_url_map_testcase.RpcDistributionStats( json_lb_stats) self.assertEqual(1, rpc_distribution.num_peers) new_backend_inuse = list( rpc_distribution.raw['rpcsByPeer'].keys())[0] self.assertNotEqual(new_backend_inuse, first_backend_inuse)