def test_udp_request_fails(self, kube_apis, crd_ingress_controller,
transport_server_setup, file):
patch_src = f"{TEST_DATA}/transport-server-udp-load-balance/{file}"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
# 4s includes 3s timeout for a health check to fail in case a backend pod doesn't respond or responds with
# an unexpected response
wait_before_test()
port = transport_server_setup.public_endpoint.udp_server_port
host = transport_server_setup.public_endpoint.public_ip
print(f"sending udp requests to: {host}:{port}")
for i in range(3):
client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
client.settimeout(2)
client.sendto("ping".encode('utf-8'), (host, port))
try:
client.recvfrom(4096)
# it should timeout
print(
f"incorrect config from {file} should have resulted in an error"
)
assert False
except socket.timeout:
print("successfully timed out")
client.close()
self.restore_ts(kube_apis, transport_server_setup)
def test_tcp_request_load_balanced_missing_service(self, kube_apis,
crd_ingress_controller,
transport_server_setup):
"""
Requests to the load balanced TCP service should result in responses from 3 different endpoints.
"""
patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/missing-service-transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
wait_before_test()
port = transport_server_setup.public_endpoint.tcp_server_port
host = transport_server_setup.public_endpoint.public_ip
print(f"sending tcp requests to: {host}:{port}")
for i in range(3):
try:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((host, port))
client.sendall(b'connect')
except ConnectionResetError as E:
print("The expected exception occurred:", E)
self.restore_ts(kube_apis, transport_server_setup)
def test_status_invalid(
self,
kube_apis,
crd_ingress_controller,
transport_server_setup,
):
"""
Test TransportServer status with an invalid protocol.
"""
patch_src = f"{TEST_DATA}/transport-server-status/rejected-invalid.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
wait_before_test()
response = read_ts(
kube_apis.custom_objects,
transport_server_setup.namespace,
transport_server_setup.name,
)
self.restore_ts(kube_apis, transport_server_setup)
assert (response["status"]
and response["status"]["reason"] == "Rejected"
and response["status"]["state"] == "Invalid")
def test_snippets(self, kube_apis, crd_ingress_controller,
transport_server_setup,
ingress_controller_prerequisites):
"""
Test snippets are present in conf when enabled
"""
patch_src = f"{TEST_DATA}/transport-server/transport-server-snippets.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
wait_before_test()
conf = get_ts_nginx_template_conf(
kube_apis.v1, transport_server_setup.namespace,
transport_server_setup.name,
transport_server_setup.ingress_pod_name,
ingress_controller_prerequisites.namespace)
print(conf)
std_src = f"{TEST_DATA}/transport-server-status/standard/transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
std_src,
transport_server_setup.namespace,
)
assert ("limit_conn_zone $binary_remote_addr zone=addr:10m;" in
conf # stream-snippets
and "limit_conn addr 1;" in conf # server-snippets
)
def test_tcp_request_load_balanced_missing_service(
self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites
):
"""
Requests to the load balanced TCP service should result in responses from 3 different endpoints.
"""
patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/missing-service-transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
wait_before_test()
port = transport_server_setup.public_endpoint.tcp_server_port
host = transport_server_setup.public_endpoint.public_ip
print(f"sending tcp requests to: {host}:{port}")
for i in range(3):
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((host, port))
response = client.recv(4096)
endpoint = response.decode()
assert endpoint == ""
client.close()
self.restore_ts(kube_apis, transport_server_setup)
def test_udp_failing_healthcheck_with_match(
self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites
):
"""
Configure a failing health check and check that NGINX Plus doesn't respond.
"""
# Step 1 - configure a failing health check
patch_src = f"{TEST_DATA}/transport-server-udp-load-balance/failing-hc-transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
wait_before_test(4)
result_conf = get_ts_nginx_template_conf(
kube_apis.v1,
transport_server_setup.namespace,
transport_server_setup.name,
transport_server_setup.ingress_pod_name,
ingress_controller_prerequisites.namespace
)
match = f"match_ts_{transport_server_setup.namespace}_transport-server_udp-app"
assert "health_check interval=5s port=3334" in result_conf
assert f"passes=1 jitter=0s fails=1 udp match={match}" in result_conf
assert "health_check_timeout 3s;"
assert 'send "health"' in result_conf
assert 'expect "unmatched"' in result_conf
# Step 2 - confirm load balancing doesn't work
port = transport_server_setup.public_endpoint.udp_server_port
host = transport_server_setup.public_endpoint.public_ip
client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
client.settimeout(2)
client.sendto("ping".encode('utf-8'), (host, port))
try:
# client.recvfrom(4096)
data, address = client.recvfrom(4096)
endpoint = data.decode()
print(f' req number response: {endpoint}')
# it should timeout
pytest.fail("expected a timeout")
except socket.timeout:
print("successfully timed out")
client.close()
# Step 3 - restore
self.restore_ts(kube_apis, transport_server_setup)
def test_tcp_failing_healthcheck_with_match(
self, kube_apis, crd_ingress_controller, transport_server_setup,
ingress_controller_prerequisites):
"""
Configure a failing health check and check that NGINX Plus resets connections.
"""
# Step 1 - configure a failing health check
patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/failing-hc-transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
# 4s includes 3s timeout for a health check to fail in case of a connection timeout to a backend pod
wait_before_test(4)
result_conf = get_ts_nginx_template_conf(
kube_apis.v1, transport_server_setup.namespace,
transport_server_setup.name,
transport_server_setup.ingress_pod_name,
ingress_controller_prerequisites.namespace)
match = f"match_ts_{transport_server_setup.namespace}_transport-server_tcp-app"
assert "health_check interval=5s port=3333" in result_conf
assert f"passes=1 jitter=0s fails=1 match={match}" in result_conf
assert "health_check_timeout 3s"
assert 'send "health"' in result_conf
assert 'expect "unmatched"' in result_conf
# Step 2 - confirm load balancing doesn't work
port = transport_server_setup.public_endpoint.tcp_server_port
host = transport_server_setup.public_endpoint.public_ip
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((host, port))
client.sendall(b'connect')
try:
client.recv(4096) # must return ConnectionResetError
client.close()
pytest.fail(
"We expected an error here, but didn't get it. Exiting...")
except ConnectionResetError as ex:
# expected error
print(f"There was an expected exception {str(ex)}")
# Step 3 - restore
self.restore_ts(kube_apis, transport_server_setup)
def restore_ts(self, kube_apis, transport_server_setup) -> None:
"""
Function to revert a TransportServer resource to a valid state.
"""
patch_src = f"{TEST_DATA}/transport-server-udp-load-balance/standard/transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
def test_total_metrics(self, crd_ingress_controller, ts_setup,
ingress_controller_endpoint, kube_apis,
test_namespace, ts):
"""
Tests nginx_ingress_controller_transportserver_resources_total metric for a given TransportServer type.
"""
ts_file = ts[0]
ts_type = ts[1]
# initially, the number of TransportServers is 0
assert_ts_total_metric(ingress_controller_endpoint, ts_type, 0)
# create a TS and check the metric is 1
ts_resource = create_ts_from_yaml(kube_apis.custom_objects, ts_file,
test_namespace)
wait_before_test()
assert_ts_total_metric(ingress_controller_endpoint, ts_type, 1)
# make the TS invalid and check the metric is 0
ts_resource["spec"]["listener"]["protocol"] = "invalid"
patch_ts(kube_apis.custom_objects, test_namespace, ts_resource)
wait_before_test()
assert_ts_total_metric(ingress_controller_endpoint, ts_type, 0)
# restore the TS and check the metric is 1
patch_ts_from_yaml(kube_apis.custom_objects,
ts_resource["metadata"]["name"], ts_file,
test_namespace)
wait_before_test()
assert_ts_total_metric(ingress_controller_endpoint, ts_type, 1)
# delete the TS and check the metric is 0
delete_ts(kube_apis.custom_objects, ts_resource, test_namespace)
wait_before_test()
assert_ts_total_metric(ingress_controller_endpoint, ts_type, 0)
def test_configurble_timeout_directives(self, kube_apis,
crd_ingress_controller,
transport_server_setup,
ingress_controller_prerequisites):
"""
Test session and upstream configurable timeouts are present in conf
"""
patch_src = f"{TEST_DATA}/transport-server/transport-server-configurable-timeouts.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
wait_before_test()
conf = get_ts_nginx_template_conf(
kube_apis.v1, transport_server_setup.namespace,
transport_server_setup.name,
transport_server_setup.ingress_pod_name,
ingress_controller_prerequisites.namespace)
print(conf)
std_src = f"{TEST_DATA}/transport-server-status/standard/transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
std_src,
transport_server_setup.namespace,
)
assert ("proxy_timeout 2s;" in conf # sessionParameters
)
assert ("proxy_connect_timeout 5s;" in conf # upstreamParameters
and "proxy_next_upstream on;" in conf
and "proxy_next_upstream_timeout 4s;" in conf
and "proxy_next_upstream_tries 3;" in conf)
def test_tcp_passing_healthcheck_with_match(
self, kube_apis, crd_ingress_controller, transport_server_setup,
ingress_controller_prerequisites):
"""
Configure a passing health check and check that all backend pods return responses.
"""
# Step 1 - configure a passing health check
patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/passing-hc-transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
# 4s includes 3s timeout for a health check to fail in case of a connection timeout to a backend pod
wait_before_test(4)
result_conf = get_ts_nginx_template_conf(
kube_apis.v1, transport_server_setup.namespace,
transport_server_setup.name,
transport_server_setup.ingress_pod_name,
ingress_controller_prerequisites.namespace)
match = f"match_ts_{transport_server_setup.namespace}_transport-server_tcp-app"
assert "health_check interval=5s port=3333" in result_conf
assert f"passes=1 jitter=0s fails=1 match={match}" in result_conf
assert "health_check_timeout 3s;"
assert 'send "health"' in result_conf
assert 'expect "healthy"' in result_conf
# Step 2 - confirm load balancing works
port = transport_server_setup.public_endpoint.tcp_server_port
host = transport_server_setup.public_endpoint.public_ip
endpoints = {}
retry = 0
while (len(endpoints) is not 3 and retry <= 30):
for i in range(20):
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((host, port))
client.sendall(b'connect')
response = client.recv(4096)
endpoint = response.decode()
print(f' req number {i}; response: {endpoint}')
if endpoint not in endpoints:
endpoints[endpoint] = 1
else:
endpoints[endpoint] = endpoints[endpoint] + 1
client.close()
retry += 1
wait_before_test(1)
print(f"Retry #{retry}")
assert len(endpoints) is 3
# Step 3 - restore
self.restore_ts(kube_apis, transport_server_setup)
def test_tcp_request_load_balanced_method(
self, kube_apis, crd_ingress_controller, transport_server_setup,
ingress_controller_prerequisites):
"""
Update load balancing method to 'hash'. This send requests to a specific pod based on it's IP. In this case
resulting in a single endpoint handling all the requests.
"""
# Step 1 - set the load balancing method.
patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/method-transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
wait_before_test()
num_servers = 0
retry = 0
while (num_servers is not 3 and retry <= 30):
result_conf = get_ts_nginx_template_conf(
kube_apis.v1, transport_server_setup.namespace,
transport_server_setup.name,
transport_server_setup.ingress_pod_name,
ingress_controller_prerequisites.namespace)
pattern = 'server .*;'
num_servers = len(re.findall(pattern, result_conf))
retry += 1
wait_before_test(1)
print(f"Retry #{retry}")
assert num_servers is 3
# Step 2 - confirm all request go to the same endpoint.
port = transport_server_setup.public_endpoint.tcp_server_port
host = transport_server_setup.public_endpoint.public_ip
endpoints = {}
retry = 0
while (len(endpoints) is not 1 and retry <= 30):
for i in range(20):
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((host, port))
client.sendall(b'connect')
response = client.recv(4096)
endpoint = response.decode()
print(f' req number {i}; response: {endpoint}')
if endpoint not in endpoints:
endpoints[endpoint] = 1
else:
endpoints[endpoint] = endpoints[endpoint] + 1
client.close()
retry += 1
wait_before_test(1)
print(f"Retry #{retry}")
assert len(endpoints) is 1
# Step 3 - restore to default load balancing method and confirm requests are balanced.
self.restore_ts(kube_apis, transport_server_setup)
wait_before_test()
endpoints = {}
retry = 0
while (len(endpoints) is not 3 and retry <= 30):
for i in range(20):
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((host, port))
client.sendall(b'connect')
response = client.recv(4096)
endpoint = response.decode()
print(f' req number {i}; response: {endpoint}')
if endpoint not in endpoints:
endpoints[endpoint] = 1
else:
endpoints[endpoint] = endpoints[endpoint] + 1
client.close()
retry += 1
wait_before_test(1)
print(f"Retry #{retry}")
assert len(endpoints) is 3
def test_tcp_request_max_connections(self, kube_apis,
crd_ingress_controller,
transport_server_setup,
ingress_controller_prerequisites):
"""
The config, maxConns, should limit the number of open TCP connections.
3 replicas of max 2 connections is 6, so making the 7th connection will fail.
"""
# step 1 - set max connections to 2 with 1 replica
patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/max-connections-transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
wait_before_test()
configs = 0
retry = 0
while (configs is not 3 and retry <= 30):
result_conf = get_ts_nginx_template_conf(
kube_apis.v1, transport_server_setup.namespace,
transport_server_setup.name,
transport_server_setup.ingress_pod_name,
ingress_controller_prerequisites.namespace)
pattern = 'max_conns=2'
configs = len(re.findall(pattern, result_conf))
retry += 1
wait_before_test(1)
print(f"Retry #{retry}")
assert configs is 3
# step 2 - make the number of allowed connections
port = transport_server_setup.public_endpoint.tcp_server_port
host = transport_server_setup.public_endpoint.public_ip
clients = []
for i in range(6):
c = self.make_holding_connection(host, port)
clients.append(c)
# step 3 - assert the next connection fails
try:
c = self.make_holding_connection(host, port)
# making a connection should fail and throw an exception
assert c is None
except ConnectionResetError as E:
print("The expected exception occurred:", E)
for c in clients:
c.close()
# step 4 - revert to config with no max connections
patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/standard/transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
wait_before_test()
# step 5 - confirm making lots of connections doesn't cause an error
clients = []
for i in range(24):
c = self.make_holding_connection(host, port)
clients.append(c)
for c in clients:
c.close()
def test_udp_passing_healthcheck_with_match(
self, kube_apis, crd_ingress_controller, transport_server_setup,
ingress_controller_prerequisites):
"""
Configure a passing health check and check that all backend pods return responses.
"""
# Step 1 - configure a passing health check
patch_src = f"{TEST_DATA}/transport-server-udp-load-balance/passing-hc-transport-server.yaml"
patch_ts(
kube_apis.custom_objects,
transport_server_setup.name,
patch_src,
transport_server_setup.namespace,
)
# 4s includes 3s timeout for a health check to fail in case a backend pod doesn't respond or responds with
# an unexpected response
wait_before_test(4)
result_conf = get_ts_nginx_template_conf(
kube_apis.v1, transport_server_setup.namespace,
transport_server_setup.name,
transport_server_setup.ingress_pod_name,
ingress_controller_prerequisites.namespace)
match = f"match_ts_{transport_server_setup.namespace}_transport-server_udp-app"
assert "health_check interval=5s port=3334" in result_conf
assert f"passes=1 jitter=0s fails=1 udp match={match}" in result_conf
assert "health_check_timeout 3s;"
assert 'send "health"' in result_conf
assert 'expect "healthy"' in result_conf
# Step 2 - confirm load balancing works
port = transport_server_setup.public_endpoint.udp_server_port
host = transport_server_setup.public_endpoint.public_ip
print(f"sending udp requests to: {host}:{port}")
retry = 0
endpoints = {}
while (len(endpoints) is not 3 and retry <= 30):
for i in range(20):
client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
client.sendto("ping".encode('utf-8'), (host, port))
data, address = client.recvfrom(4096)
endpoint = data.decode()
print(f' req number {i}; response: {endpoint}')
if endpoint not in endpoints:
endpoints[endpoint] = 1
else:
endpoints[endpoint] = endpoints[endpoint] + 1
client.close()
retry += 1
wait_before_test(1)
print(f"Retry #{retry}")
assert len(endpoints) is 3
# Step 3 - restore
self.restore_ts(kube_apis, transport_server_setup)
