def setUpClass(cls):
"""Hook method for setting up class fixture before running tests in
the class.
"""
logger.info('----- Testing %s -----', cls.__name__)
logger.info('Logs timezone: %s', time.localtime().tm_zone)
# Raises unittest.SkipTest if given client/server/version does not
# support current test case.
cls.lang_spec = skips.evaluate_test_config(cls.is_supported)
# Must be called before KubernetesApiManager or GcpApiManager init.
xds_flags.set_socket_default_timeout_from_flag()
# GCP
cls.project = xds_flags.PROJECT.value
cls.network = xds_flags.NETWORK.value
cls.gcp_service_account = xds_k8s_flags.GCP_SERVICE_ACCOUNT.value
cls.td_bootstrap_image = xds_k8s_flags.TD_BOOTSTRAP_IMAGE.value
cls.xds_server_uri = xds_flags.XDS_SERVER_URI.value
cls.ensure_firewall = xds_flags.ENSURE_FIREWALL.value
cls.firewall_allowed_ports = xds_flags.FIREWALL_ALLOWED_PORTS.value
cls.compute_api_version = xds_flags.COMPUTE_API_VERSION.value
# Resource names.
cls.resource_prefix = xds_flags.RESOURCE_PREFIX.value
if xds_flags.RESOURCE_SUFFIX.value is not None:
cls.resource_suffix_randomize = False
cls.resource_suffix = xds_flags.RESOURCE_SUFFIX.value
# Test server
cls.server_image = xds_k8s_flags.SERVER_IMAGE.value
cls.server_name = xds_flags.SERVER_NAME.value
cls.server_port = xds_flags.SERVER_PORT.value
cls.server_maintenance_port = xds_flags.SERVER_MAINTENANCE_PORT.value
cls.server_xds_host = xds_flags.SERVER_NAME.value
cls.server_xds_port = xds_flags.SERVER_XDS_PORT.value
# Test client
cls.client_image = xds_k8s_flags.CLIENT_IMAGE.value
cls.client_name = xds_flags.CLIENT_NAME.value
cls.client_port = xds_flags.CLIENT_PORT.value
# Test suite settings
cls.force_cleanup = xds_flags.FORCE_CLEANUP.value
cls.debug_use_port_forwarding = \
xds_k8s_flags.DEBUG_USE_PORT_FORWARDING.value
cls.enable_workload_identity = \
xds_k8s_flags.ENABLE_WORKLOAD_IDENTITY.value
cls.check_local_certs = _CHECK_LOCAL_CERTS.value
# Resource managers
cls.k8s_api_manager = k8s.KubernetesApiManager(
xds_k8s_flags.KUBE_CONTEXT.value)
cls.secondary_k8s_api_manager = k8s.KubernetesApiManager(
xds_k8s_flags.SECONDARY_KUBE_CONTEXT.value)
cls.gcp_api_manager = gcp.api.GcpApiManager()
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
load_keep_config()
# Must be called before KubernetesApiManager or GcpApiManager init.
xds_flags.set_socket_default_timeout_from_flag()
project: str = xds_flags.PROJECT.value
network: str = xds_flags.NETWORK.value
gcp_service_account: str = xds_k8s_flags.GCP_SERVICE_ACCOUNT.value
dry_run: bool = DRY_RUN.value
td_resource_rules = [
# itmes in each tuple, in order
# - regex to match
# - prefix of the resource (only used by gke resources)
# - function to check of the resource should be kept
# - function to delete the resource
(r'test-hc(.*)', '', is_marked_as_keep_gce,
remove_relative_resources_run_xds_tests),
(r'test-template(.*)', '', is_marked_as_keep_gce,
remove_relative_resources_run_xds_tests),
]
for prefix in TD_RESOURCE_PREFIXES.value:
td_resource_rules.append((f'{prefix}-health-check-(.*)', prefix,
is_marked_as_keep_gke, cleanup_td_for_gke), )
# List resources older than KEEP_PERIOD. We only list health-checks and
# instance templates because these are leaves in the resource dependency tree.
#
# E.g. forwarding-rule depends on the target-proxy. So leaked
# forwarding-rule indicates there's a leaked target-proxy (because this
# target proxy cannot deleted unless the forwarding rule is deleted). The
# leaked target-proxy is guaranteed to be a super set of leaked
# forwarding-rule.
compute = gcp.compute.ComputeV1(gcp.api.GcpApiManager(), project)
leakedHealthChecks = []
for item in compute.list_health_check()['items']:
if dateutil.parser.isoparse(
item['creationTimestamp']) <= get_expire_timestamp():
leakedHealthChecks.append(item)
delete_leaked_td_resources(dry_run, td_resource_rules, project, network,
leakedHealthChecks)
# Delete leaked instance templates, those usually mean there are leaked VMs
# from the gce framework. Also note that this is only needed for the gce
# resources.
leakedInstanceTemplates = exec_gcloud(project, 'compute',
'instance-templates', 'list')
delete_leaked_td_resources(dry_run, td_resource_rules, project, network,
leakedInstanceTemplates)
find_and_remove_leaked_k8s_resources(dry_run, project, network,
gcp_service_account)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
# Must be called before KubernetesApiManager or GcpApiManager init.
xds_flags.set_socket_default_timeout_from_flag()
project: str = xds_flags.PROJECT.value
# GCP Service Account email
gcp_service_account: str = xds_k8s_flags.GCP_SERVICE_ACCOUNT.value
# KubernetesClientRunner arguments.
runner_kwargs = dict(
deployment_name=xds_flags.CLIENT_NAME.value,
image_name=xds_k8s_flags.CLIENT_IMAGE.value,
td_bootstrap_image=xds_k8s_flags.TD_BOOTSTRAP_IMAGE.value,
gcp_project=project,
gcp_api_manager=gcp.api.GcpApiManager(),
gcp_service_account=gcp_service_account,
xds_server_uri=xds_flags.XDS_SERVER_URI.value,
network=xds_flags.NETWORK.value,
stats_port=xds_flags.CLIENT_PORT.value,
reuse_namespace=_REUSE_NAMESPACE.value)
if _SECURE.value:
runner_kwargs.update(
deployment_template='client-secure.deployment.yaml')
k8s_api_manager = k8s.KubernetesApiManager(
xds_k8s_flags.KUBE_CONTEXT.value)
client_namespace = _KubernetesClientRunner.make_namespace_name(
xds_flags.RESOURCE_PREFIX.value, xds_flags.RESOURCE_SUFFIX.value)
client_runner = _KubernetesClientRunner(
k8s.KubernetesNamespace(k8s_api_manager, client_namespace),
**runner_kwargs)
# Server target
server_xds_host = xds_flags.SERVER_XDS_HOST.value
server_xds_port = xds_flags.SERVER_XDS_PORT.value
if _CMD.value == 'run':
logger.info('Run client, secure_mode=%s', _SECURE.value)
client_runner.run(
server_target=f'xds:///{server_xds_host}:{server_xds_port}',
qps=_QPS.value,
print_response=_PRINT_RESPONSE.value,
secure_mode=_SECURE.value,
config_mesh=_CONFIG_MESH.value)
elif _CMD.value == 'cleanup':
logger.info('Cleanup client')
client_runner.cleanup(force=True,
force_namespace=_CLEANUP_NAMESPACE.value)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
# Must be called before KubernetesApiManager or GcpApiManager init.
xds_flags.set_socket_default_timeout_from_flag()
project: str = xds_flags.PROJECT.value
# GCP Service Account email
gcp_service_account: str = xds_k8s_flags.GCP_SERVICE_ACCOUNT.value
# Resource names.
resource_prefix: str = xds_flags.RESOURCE_PREFIX.value
resource_suffix: str = xds_flags.RESOURCE_SUFFIX.value
# KubernetesServerRunner arguments.
runner_kwargs = dict(
deployment_name=xds_flags.SERVER_NAME.value,
image_name=xds_k8s_flags.SERVER_IMAGE.value,
td_bootstrap_image=xds_k8s_flags.TD_BOOTSTRAP_IMAGE.value,
gcp_project=project,
gcp_api_manager=gcp.api.GcpApiManager(),
gcp_service_account=gcp_service_account,
network=xds_flags.NETWORK.value,
reuse_namespace=_REUSE_NAMESPACE.value)
if _SECURE.value:
runner_kwargs.update(
xds_server_uri=xds_flags.XDS_SERVER_URI.value,
deployment_template='server-secure.deployment.yaml')
k8s_api_manager = k8s.KubernetesApiManager(xds_k8s_flags.KUBE_CONTEXT.value)
server_namespace = _KubernetesServerRunner.make_namespace_name(
resource_prefix, resource_suffix)
server_runner = _KubernetesServerRunner(
k8s.KubernetesNamespace(k8s_api_manager, server_namespace),
**runner_kwargs)
if _CMD.value == 'run':
logger.info('Run server, secure_mode=%s', _SECURE.value)
server_runner.run(
test_port=xds_flags.SERVER_PORT.value,
maintenance_port=xds_flags.SERVER_MAINTENANCE_PORT.value,
secure_mode=_SECURE.value)
elif _CMD.value == 'cleanup':
logger.info('Cleanup server')
server_runner.cleanup(force=True,
force_namespace=_CLEANUP_NAMESPACE.value)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
cleanup.load_keep_config()
# Must be called before KubernetesApiManager or GcpApiManager init.
xds_flags.set_socket_default_timeout_from_flag()
project: str = xds_flags.PROJECT.value
network: str = xds_flags.NETWORK.value
gcp_service_account: str = xds_k8s_flags.GCP_SERVICE_ACCOUNT.value
dry_run: bool = cleanup.DRY_RUN.value
cleanup.find_and_remove_leaked_k8s_resources(dry_run, project, network,
gcp_service_account)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
# Must be called before KubernetesApiManager or GcpApiManager init.
xds_flags.set_socket_default_timeout_from_flag()
k8s_api_manager = k8s.KubernetesApiManager(
xds_k8s_flags.KUBE_CONTEXT.value)
# Resource names.
resource_prefix: str = xds_flags.RESOURCE_PREFIX.value
# Server
server_name = xds_flags.SERVER_NAME.value
server_namespace = resource_prefix
server_k8s_ns = k8s.KubernetesNamespace(k8s_api_manager, server_namespace)
server_pod = get_deployment_pods(server_k8s_ns, server_name)[0]
test_server: _XdsTestServer = _XdsTestServer(
ip=server_pod.status.pod_ip,
rpc_port=xds_flags.SERVER_PORT.value,
hostname=server_pod.metadata.name,
xds_host=xds_flags.SERVER_XDS_HOST.value,
xds_port=xds_flags.SERVER_XDS_PORT.value,
rpc_host=_SERVER_RPC_HOST.value)
# Client
client_name = xds_flags.CLIENT_NAME.value
client_namespace = resource_prefix
client_k8s_ns = k8s.KubernetesNamespace(k8s_api_manager, client_namespace)
client_pod = get_deployment_pods(client_k8s_ns, client_name)[0]
test_client: _XdsTestClient = _XdsTestClient(
ip=client_pod.status.pod_ip,
rpc_port=xds_flags.CLIENT_PORT.value,
server_target=test_server.xds_uri,
hostname=client_pod.metadata.name,
rpc_host=_CLIENT_RPC_HOST.value)
if _SECURITY.value in ('mtls', 'tls', 'plaintext'):
debug_security_setup_positive(test_client, test_server)
elif _SECURITY.value == ('mtls_error', 'server_authz_error'):
debug_security_setup_negative(test_client)
else:
debug_basic_setup(test_client, test_server)
test_client.close()
test_server.close()
def main(argv): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
# Must be called before KubernetesApiManager or GcpApiManager init.
xds_flags.set_socket_default_timeout_from_flag()
command = _CMD.value
security_mode = _SECURITY.value
project: str = xds_flags.PROJECT.value
network: str = xds_flags.NETWORK.value
# Resource names.
resource_prefix: str = xds_flags.RESOURCE_PREFIX.value
resource_suffix: str = xds_flags.RESOURCE_SUFFIX.value
# Test server
server_name = xds_flags.SERVER_NAME.value
server_port = xds_flags.SERVER_PORT.value
server_maintenance_port = xds_flags.SERVER_MAINTENANCE_PORT.value
server_xds_host = xds_flags.SERVER_XDS_HOST.value
server_xds_port = xds_flags.SERVER_XDS_PORT.value
server_namespace = _KubernetesServerRunner.make_namespace_name(
resource_prefix, resource_suffix)
gcp_api_manager = gcp.api.GcpApiManager()
if security_mode is None:
td = traffic_director.TrafficDirectorManager(
gcp_api_manager,
project=project,
network=network,
resource_prefix=resource_prefix,
resource_suffix=resource_suffix)
else:
td = traffic_director.TrafficDirectorSecureManager(
gcp_api_manager,
project=project,
network=network,
resource_prefix=resource_prefix,
resource_suffix=resource_suffix)
if server_maintenance_port is None:
server_maintenance_port = \
_KubernetesServerRunner.DEFAULT_SECURE_MODE_MAINTENANCE_PORT
try:
if command in ('create', 'cycle'):
logger.info('Create mode')
if security_mode is None:
logger.info('No security')
td.setup_for_grpc(server_xds_host,
server_xds_port,
health_check_port=server_maintenance_port)
elif security_mode == 'mtls':
logger.info('Setting up mtls')
td.setup_for_grpc(server_xds_host,
server_xds_port,
health_check_port=server_maintenance_port)
td.setup_server_security(server_namespace=server_namespace,
server_name=server_name,
server_port=server_port,
tls=True,
mtls=True)
td.setup_client_security(server_namespace=server_namespace,
server_name=server_name,
tls=True,
mtls=True)
elif security_mode == 'tls':
logger.info('Setting up tls')
td.setup_for_grpc(server_xds_host,
server_xds_port,
health_check_port=server_maintenance_port)
td.setup_server_security(server_namespace=server_namespace,
server_name=server_name,
server_port=server_port,
tls=True,
mtls=False)
td.setup_client_security(server_namespace=server_namespace,
server_name=server_name,
tls=True,
mtls=False)
elif security_mode == 'plaintext':
logger.info('Setting up plaintext')
td.setup_for_grpc(server_xds_host,
server_xds_port,
health_check_port=server_maintenance_port)
td.setup_server_security(server_namespace=server_namespace,
server_name=server_name,
server_port=server_port,
tls=False,
mtls=False)
td.setup_client_security(server_namespace=server_namespace,
server_name=server_name,
tls=False,
mtls=False)
elif security_mode == 'mtls_error':
# Error case: server expects client mTLS cert,
# but client configured only for TLS
logger.info('Setting up mtls_error')
td.setup_for_grpc(server_xds_host,
server_xds_port,
health_check_port=server_maintenance_port)
td.setup_server_security(server_namespace=server_namespace,
server_name=server_name,
server_port=server_port,
tls=True,
mtls=True)
td.setup_client_security(server_namespace=server_namespace,
server_name=server_name,
tls=True,
mtls=False)
elif security_mode == 'server_authz_error':
# Error case: client does not authorize server
# because of mismatched SAN name.
logger.info('Setting up mtls_error')
td.setup_for_grpc(server_xds_host,
server_xds_port,
health_check_port=server_maintenance_port)
# Regular TLS setup, but with client policy configured using
# intentionality incorrect server_namespace.
td.setup_server_security(server_namespace=server_namespace,
server_name=server_name,
server_port=server_port,
tls=True,
mtls=False)
td.setup_client_security(
server_namespace=f'incorrect-namespace-{rand.rand_string()}',
server_name=server_name,
tls=True,
mtls=False)
logger.info('Works!')
except Exception: # noqa pylint: disable=broad-except
logger.exception('Got error during creation')
if command in ('cleanup', 'cycle'):
logger.info('Cleaning up')
td.cleanup(force=True)
if command == 'backends-add':
logger.info('Adding backends')
k8s_api_manager = k8s.KubernetesApiManager(
xds_k8s_flags.KUBE_CONTEXT.value)
k8s_namespace = k8s.KubernetesNamespace(k8s_api_manager,
server_namespace)
neg_name, neg_zones = k8s_namespace.get_service_neg(
server_name, server_port)
td.load_backend_service()
td.backend_service_add_neg_backends(neg_name, neg_zones)
td.wait_for_backends_healthy_status()
elif command == 'backends-cleanup':
td.load_backend_service()
td.backend_service_remove_all_backends()
elif command == 'unused-xds-port':
try:
unused_xds_port = td.find_unused_forwarding_rule_port()
logger.info('Found unused forwarding rule port: %s',
unused_xds_port)
except Exception: # noqa pylint: disable=broad-except
logger.exception("Couldn't find unused forwarding rule port")
def __init__(self, absl_flags: Mapping[str, Any] = None):
if absl_flags is not None:
for key in absl_flags:
setattr(self, key, absl_flags[key])
# Pick a client_namespace_suffix if not set
if getattr(self, 'resource_suffix', None) is None:
self.resource_suffix = ""
else:
raise NotImplementedError(
'Predefined resource_suffix is not supported for UrlMap tests')
logging.info('GcpResourceManager: resource prefix=%s, suffix=%s',
self.resource_prefix, self.resource_suffix)
# Must be called before KubernetesApiManager or GcpApiManager init.
xds_flags.set_socket_default_timeout_from_flag()
# API managers
self.k8s_api_manager = k8s.KubernetesApiManager(self.kube_context)
self.gcp_api_manager = gcp.api.GcpApiManager()
self.td = traffic_director.TrafficDirectorManager(
self.gcp_api_manager,
self.project,
resource_prefix=self.resource_prefix,
resource_suffix=(self.resource_suffix or ""),
network=self.network,
compute_api_version=self.compute_api_version,
)
# Kubernetes namespace
self.k8s_namespace = k8s.KubernetesNamespace(self.k8s_api_manager,
self.resource_prefix)
# Kubernetes Test Servers
self.test_server_runner = _KubernetesServerRunner(
self.k8s_namespace,
deployment_name=self.server_name,
image_name=self.server_image,
gcp_project=self.project,
gcp_api_manager=self.gcp_api_manager,
gcp_service_account=self.gcp_service_account,
td_bootstrap_image=self.td_bootstrap_image,
xds_server_uri=self.xds_server_uri,
network=self.network,
enable_workload_identity=self.enable_workload_identity)
self.test_server_alternative_runner = _KubernetesServerRunner(
self.k8s_namespace,
deployment_name=self.server_name + '-alternative',
image_name=self.server_image,
gcp_project=self.project,
gcp_api_manager=self.gcp_api_manager,
gcp_service_account=self.gcp_service_account,
td_bootstrap_image=self.td_bootstrap_image,
xds_server_uri=self.xds_server_uri,
network=self.network,
enable_workload_identity=self.enable_workload_identity,
reuse_namespace=True)
self.test_server_affinity_runner = _KubernetesServerRunner(
self.k8s_namespace,
deployment_name=self.server_name + '-affinity',
image_name=self.server_image,
gcp_project=self.project,
gcp_api_manager=self.gcp_api_manager,
gcp_service_account=self.gcp_service_account,
td_bootstrap_image=self.td_bootstrap_image,
xds_server_uri=self.xds_server_uri,
network=self.network,
enable_workload_identity=self.enable_workload_identity,
reuse_namespace=True)
logging.info('Strategy of GCP resources management: %s', self.strategy)