def test_subsetting_basic(self) -> None:
with self.subTest('00_create_health_check'):
self.td.create_health_check()
with self.subTest('01_create_backend_services'):
self.td.create_backend_service(subset_size=_SUBSET_SIZE)
with self.subTest('02_create_url_map'):
self.td.create_url_map(self.server_xds_host, self.server_xds_port)
with self.subTest('03_create_target_proxy'):
self.td.create_target_proxy()
with self.subTest('04_create_forwarding_rule'):
self.td.create_forwarding_rule(self.server_xds_port)
test_servers: List[_XdsTestServer]
with self.subTest('05_start_test_servers'):
test_servers = self.startTestServers(replica_count=_NUM_BACKENDS)
with self.subTest('06_add_server_backends_to_backend_services'):
self.setupServerBackends()
rpc_distribution = collections.defaultdict(int)
with self.subTest('07_start_test_client'):
for i in range(_NUM_CLIENTS):
# Clean created client pods if there is any
self.client_runner.cleanup(force=True)
# Create a test client
test_client: _XdsTestClient = self.startTestClient(
test_servers[0])
# Validate the number of received endpoints
config = test_client.csds.fetch_client_status(
log_level=logging.INFO)
self.assertIsNotNone(config)
json_config = json_format.MessageToDict(config)
parsed = xds_url_map_testcase.DumpedXdsConfig(json_config)
logging.info('Client %d received endpoints (len=%s): %s', i,
len(parsed.endpoints), parsed.endpoints)
self.assertLen(parsed.endpoints, _SUBSET_SIZE)
# Record RPC stats
lb_stats = self.getClientRpcStats(test_client,
_NUM_BACKENDS * 25)
for key, value in lb_stats.rpcs_by_peer.items():
rpc_distribution[key] += value
with self.subTest('08_log_rpc_distribution'):
server_entries = sorted(rpc_distribution.items(),
key=lambda x: -x[1])
# Validate if clients are receiving different sets of backends (3
# client received a total of 4 unique backends == FAIL, a total of 5
# unique backends == PASS)
self.assertGreater(len(server_entries), _SUBSET_SIZE)
logging.info('RPC distribution (len=%s): %s', len(server_entries),
server_entries)
peak = server_entries[0][1]
mean = sum(map(lambda x: x[1],
server_entries)) / len(server_entries)
logging.info('Peak=%d Mean=%.1f Peak-to-Mean-Ratio=%.2f', peak,
mean, peak / mean)
def assertRouteConfigUpdateTrafficHandoff(
self, test_client: XdsTestClient,
previous_route_config_version: str, retry_wait_second: int,
timeout_second: int):
retryer = retryers.constant_retryer(
wait_fixed=datetime.timedelta(seconds=retry_wait_second),
timeout=datetime.timedelta(seconds=timeout_second),
retry_on_exceptions=(TdPropagationRetryableError, ),
logger=logger,
log_level=logging.INFO)
try:
for attempt in retryer:
with attempt:
self.assertSuccessfulRpcs(test_client)
raw_config = test_client.csds.fetch_client_status(
log_level=logging.INFO)
dumped_config = xds_url_map_testcase.DumpedXdsConfig(
json_format.MessageToDict(raw_config))
route_config_version = dumped_config.rds_version
if previous_route_config_version == route_config_version:
logger.info(
'Routing config not propagated yet. Retrying.')
raise TdPropagationRetryableError(
"CSDS not get updated routing config corresponding"
" to the second set of url maps")
else:
self.assertSuccessfulRpcs(test_client)
logger.info(
'[SUCCESS] Confirmed successful RPC with the updated routing config, version=%s',
route_config_version)
except retryers.RetryError as retry_error:
logger.info(
'Retry exhausted. TD routing config propagation failed after timeout %ds. Last seen client config dump: %s',
timeout_second, dumped_config)
raise retry_error
def test_affinity(self) -> None: # pylint: disable=too-many-statements
with self.subTest('00_create_health_check'):
self.td.create_health_check()
with self.subTest('01_create_backend_services'):
self.td.create_backend_service(
affinity_header=_TEST_AFFINITY_METADATA_KEY)
with self.subTest('02_create_url_map'):
self.td.create_url_map(self.server_xds_host, self.server_xds_port)
with self.subTest('03_create_target_proxy'):
self.td.create_target_proxy()
with self.subTest('04_create_forwarding_rule'):
self.td.create_forwarding_rule(self.server_xds_port)
test_servers: List[_XdsTestServer]
with self.subTest('05_start_test_servers'):
test_servers = self.startTestServers(replica_count=_REPLICA_COUNT)
with self.subTest('06_add_server_backends_to_backend_services'):
self.setupServerBackends()
test_client: _XdsTestClient
with self.subTest('07_start_test_client'):
test_client = self.startTestClient(test_servers[0],
rpc='EmptyCall',
metadata='EmptyCall:%s:123' %
_TEST_AFFINITY_METADATA_KEY)
# Validate the number of received endpoints and affinity configs.
config = test_client.csds.fetch_client_status(
log_level=logging.INFO)
self.assertIsNotNone(config)
json_config = json_format.MessageToDict(config)
parsed = xds_url_map_testcase.DumpedXdsConfig(json_config)
logging.info('Client received CSDS response: %s', parsed)
self.assertLen(parsed.endpoints, _REPLICA_COUNT)
self.assertEqual(
parsed.rds['virtualHosts'][0]['routes'][0]['route']
['hashPolicy'][0]['header']['headerName'],
_TEST_AFFINITY_METADATA_KEY)
self.assertEqual(parsed.cds[0]['lbPolicy'], 'RING_HASH')
with self.subTest('08_test_client_xds_config_exists'):
self.assertXdsConfigExists(test_client)
with self.subTest('09_test_server_received_rpcs_from_test_client'):
self.assertSuccessfulRpcs(test_client)
with self.subTest('10_first_100_affinity_rpcs_pick_same_backend'):
rpc_stats = self.getClientRpcStats(test_client, _RPC_COUNT)
json_lb_stats = json_format.MessageToDict(rpc_stats)
rpc_distribution = xds_url_map_testcase.RpcDistributionStats(
json_lb_stats)
self.assertEqual(1, rpc_distribution.num_peers)
self.assertLen(
test_client.find_subchannels_with_state(
_ChannelzChannelState.READY),
1,
)
self.assertLen(
test_client.find_subchannels_with_state(
_ChannelzChannelState.IDLE),
2,
)
# Remember the backend inuse, and turn it down later.
first_backend_inuse = list(
rpc_distribution.raw['rpcsByPeer'].keys())[0]
with self.subTest('11_turn_down_server_in_use'):
for server in test_servers:
if server.hostname == first_backend_inuse:
server.set_not_serving()
with self.subTest('12_wait_for_unhealth_status_propagation'):
deadline = time.time() + _TD_PROPAGATE_TIMEOUT
parsed = None
try:
while time.time() < deadline:
config = test_client.csds.fetch_client_status(
log_level=logging.INFO)
self.assertIsNotNone(config)
json_config = json_format.MessageToDict(config)
parsed = xds_url_map_testcase.DumpedXdsConfig(json_config)
if len(parsed.endpoints) == _REPLICA_COUNT - 1:
break
logging.info(
'CSDS got unexpected endpoints, will retry after %d seconds',
_TD_PROPAGATE_CHECK_INTERVAL_SEC)
time.sleep(_TD_PROPAGATE_CHECK_INTERVAL_SEC)
else:
self.fail(
'unhealthy status did not propagate after 600 seconds')
finally:
logging.info('Client received CSDS response: %s', parsed)
with self.subTest('12_next_100_affinity_rpcs_pick_different_backend'):
rpc_stats = self.getClientRpcStats(test_client, _RPC_COUNT)
json_lb_stats = json_format.MessageToDict(rpc_stats)
rpc_distribution = xds_url_map_testcase.RpcDistributionStats(
json_lb_stats)
self.assertEqual(1, rpc_distribution.num_peers)
new_backend_inuse = list(
rpc_distribution.raw['rpcsByPeer'].keys())[0]
self.assertNotEqual(new_backend_inuse, first_backend_inuse)
