def test_number_of_replicas(self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites): """ The load balancing of UDP should result in 4 servers to match the 4 replicas of a service. """ original = scale_deployment(kube_apis.apps_v1_api, "udp-service", transport_server_setup.namespace, 4) wait_before_test() result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace) print(result_conf) pattern = 'server .*;' num_servers = len(re.findall(pattern, result_conf)) assert num_servers is 4 scale_deployment(kube_apis.apps_v1_api, "udp-service", transport_server_setup.namespace, original) wait_before_test()
def test_snippets(self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites): """ Test snippets are present in conf when enabled """ patch_src = f"{TEST_DATA}/transport-server/transport-server-snippets.yaml" patch_ts( kube_apis.custom_objects, transport_server_setup.name, patch_src, transport_server_setup.namespace, ) wait_before_test() conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace) print(conf) std_src = f"{TEST_DATA}/transport-server-status/standard/transport-server.yaml" patch_ts( kube_apis.custom_objects, transport_server_setup.name, std_src, transport_server_setup.namespace, ) assert ("limit_conn_zone $binary_remote_addr zone=addr:10m;" in conf # stream-snippets and "limit_conn addr 1;" in conf # server-snippets )
def test_number_of_replicas(self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites): """ The load balancing of TCP should result in 4 servers to match the 4 replicas of a service. """ original = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "tcp-service", transport_server_setup.namespace, 4) num_servers = 0 retry = 0 while (num_servers is not 4 and retry <= 30): result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace) pattern = 'server .*;' num_servers = len(re.findall(pattern, result_conf)) retry += 1 wait_before_test(1) print(f"Retry #{retry}") assert num_servers is 4 scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "tcp-service", transport_server_setup.namespace, original) retry = 0 while (num_servers is not original and retry <= 50): result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace) pattern = 'server .*;' num_servers = len(re.findall(pattern, result_conf)) retry += 1 wait_before_test(1) print(f"Retry #{retry}") assert num_servers is original
def test_udp_failing_healthcheck_with_match( self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites ): """ Configure a failing health check and check that NGINX Plus doesn't respond. """ # Step 1 - configure a failing health check patch_src = f"{TEST_DATA}/transport-server-udp-load-balance/failing-hc-transport-server.yaml" patch_ts_from_yaml( kube_apis.custom_objects, transport_server_setup.name, patch_src, transport_server_setup.namespace, ) wait_before_test(4) result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace ) match = f"match_ts_{transport_server_setup.namespace}_transport-server_udp-app" assert "health_check interval=5s" in result_conf assert f"passes=1 jitter=0s fails=1 udp match={match}" in result_conf assert "health_check_timeout 3s;" assert 'send "health"' in result_conf assert 'expect "unmatched"' in result_conf # Step 2 - confirm load balancing doesn't work port = transport_server_setup.public_endpoint.udp_server_port host = transport_server_setup.public_endpoint.public_ip host = host.strip("[]") client = socket.socket(ipfamily_from_host(host), socket.SOCK_DGRAM, 0) client.settimeout(2) client.sendto("ping".encode('utf-8'), (host, port)) try: # client.recvfrom(4096) data, address = client.recvfrom(4096) endpoint = data.decode() print(f' req number response: {endpoint}') # it should timeout pytest.fail("expected a timeout") except socket.timeout: print("successfully timed out") client.close() # Step 3 - restore self.restore_ts(kube_apis, transport_server_setup)
def test_tcp_failing_healthcheck_with_match( self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites): """ Configure a failing health check and check that NGINX Plus resets connections. """ # Step 1 - configure a failing health check patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/failing-hc-transport-server.yaml" patch_ts_from_yaml( kube_apis.custom_objects, transport_server_setup.name, patch_src, transport_server_setup.namespace, ) # 4s includes 3s timeout for a health check to fail in case of a connection timeout to a backend pod wait_before_test(4) result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace) match = f"match_ts_{transport_server_setup.namespace}_transport-server_tcp-app" assert "health_check interval=5s" in result_conf assert f"passes=1 jitter=0s fails=1 match={match}" in result_conf assert "health_check_timeout 3s" assert 'send "health"' in result_conf assert 'expect "unmatched"' in result_conf # Step 2 - confirm load balancing doesn't work port = transport_server_setup.public_endpoint.tcp_server_port host = transport_server_setup.public_endpoint.public_ip host = host.strip("[]") client = socket.create_connection((host, port)) client.sendall(b'connect') try: client.recv(4096) # must return ConnectionResetError client.close() pytest.fail( "We expected an error here, but didn't get it. Exiting...") except ConnectionResetError as ex: # expected error print(f"There was an expected exception {str(ex)}") # Step 3 - restore self.restore_ts(kube_apis, transport_server_setup)
def test_udp_request_load_balanced( self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites ): """ Requests to the load balanced UDP service should result in responses from 3 different endpoints. """ wait_before_test() port = transport_server_setup.public_endpoint.udp_server_port host = transport_server_setup.public_endpoint.public_ip print(f"sending udp requests to: {host}:{port}") endpoints = {} retry = 0 while(len(endpoints) is not 3 and retry <= 30): for i in range(20): host = host.strip("[]") client = socket.socket(ipfamily_from_host(host), socket.SOCK_DGRAM, 0) client.sendto("ping".encode('utf-8'), (host, port)) data, address = client.recvfrom(4096) endpoint = data.decode() print(f' req number {i}; response: {endpoint}') if endpoint not in endpoints: endpoints[endpoint] = 1 else: endpoints[endpoint] = endpoints[endpoint] + 1 client.close() retry += 1 wait_before_test(1) print(f"Retry #{retry}") assert len(endpoints) is 3 result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace ) pattern = 'server .*;' servers = re.findall(pattern, result_conf) for key in endpoints.keys(): found = False for server in servers: if chk_endpoint(key) in server: found = True assert found
def test_tcp_request_load_balanced(self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites): """ Requests to the load balanced TCP service should result in responses from 3 different endpoints. """ wait_before_test() port = transport_server_setup.public_endpoint.tcp_server_port host = transport_server_setup.public_endpoint.public_ip print(f"sending tcp requests to: {host}:{port}") endpoints = {} retry = 0 while (len(endpoints) is not 3 and retry <= 30): for i in range(20): host = host.strip("[]") client = socket.create_connection((host, port)) client.sendall(b'connect') response = client.recv(4096) endpoint = response.decode() print(f' req number {i}; response: {endpoint}') if endpoint not in endpoints: endpoints[endpoint] = 1 else: endpoints[endpoint] = endpoints[endpoint] + 1 client.close() retry += 1 wait_before_test(1) print(f"Retry #{retry}") assert len(endpoints) is 3 result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace) pattern = 'server .*;' servers = re.findall(pattern, result_conf) for key in endpoints.keys(): found = False for server in servers: if key in server: found = True assert found
def test_configurble_timeout_directives( self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites ): """ Test session and upstream configurable timeouts are present in conf """ patch_src = f"{TEST_DATA}/transport-server/transport-server-configurable-timeouts.yaml" patch_ts_from_yaml( kube_apis.custom_objects, transport_server_setup.name, patch_src, transport_server_setup.namespace, ) wait_before_test() conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace ) print(conf) std_src = f"{TEST_DATA}/transport-server-status/standard/transport-server.yaml" patch_ts_from_yaml( kube_apis.custom_objects, transport_server_setup.name, std_src, transport_server_setup.namespace, ) assert ( "proxy_timeout 2s;" in conf # sessionParameters ) assert ( "proxy_connect_timeout 5s;" in conf # upstreamParameters and "proxy_next_upstream on;" in conf and "proxy_next_upstream_timeout 4s;" in conf and "proxy_next_upstream_tries 3;" in conf )
def test_tcp_passing_healthcheck_with_match( self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites): """ Configure a passing health check and check that all backend pods return responses. """ # Step 1 - configure a passing health check patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/passing-hc-transport-server.yaml" patch_ts_from_yaml( kube_apis.custom_objects, transport_server_setup.name, patch_src, transport_server_setup.namespace, ) # 4s includes 3s timeout for a health check to fail in case of a connection timeout to a backend pod wait_before_test(4) result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace) match = f"match_ts_{transport_server_setup.namespace}_transport-server_tcp-app" assert "health_check interval=5s" in result_conf assert f"passes=1 jitter=0s fails=1 match={match}" in result_conf assert "health_check_timeout 3s;" assert 'send "health"' in result_conf assert 'expect "healthy"' in result_conf # Step 2 - confirm load balancing works port = transport_server_setup.public_endpoint.tcp_server_port host = transport_server_setup.public_endpoint.public_ip endpoints = {} retry = 0 while (len(endpoints) is not 3 and retry <= 30): for i in range(20): host = host.strip("[]") client = socket.create_connection((host, port)) client.sendall(b'connect') response = client.recv(4096) endpoint = response.decode() print(f' req number {i}; response: {endpoint}') if endpoint not in endpoints: endpoints[endpoint] = 1 else: endpoints[endpoint] = endpoints[endpoint] + 1 client.close() retry += 1 wait_before_test(1) print(f"Retry #{retry}") assert len(endpoints) is 3 # Step 3 - restore self.restore_ts(kube_apis, transport_server_setup)
def test_tcp_request_load_balanced_method( self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites): """ Update load balancing method to 'hash'. This send requests to a specific pod based on it's IP. In this case resulting in a single endpoint handling all the requests. """ # Step 1 - set the load balancing method. patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/method-transport-server.yaml" patch_ts_from_yaml( kube_apis.custom_objects, transport_server_setup.name, patch_src, transport_server_setup.namespace, ) wait_before_test() num_servers = 0 retry = 0 while (num_servers is not 3 and retry <= 30): result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace) pattern = 'server .*;' num_servers = len(re.findall(pattern, result_conf)) retry += 1 wait_before_test(1) print(f"Retry #{retry}") assert num_servers is 3 # Step 2 - confirm all request go to the same endpoint. port = transport_server_setup.public_endpoint.tcp_server_port host = transport_server_setup.public_endpoint.public_ip endpoints = {} retry = 0 while (len(endpoints) is not 1 and retry <= 30): for i in range(20): host = host.strip("[]") client = socket.create_connection((host, port)) client.sendall(b'connect') response = client.recv(4096) endpoint = response.decode() print(f' req number {i}; response: {endpoint}') if endpoint not in endpoints: endpoints[endpoint] = 1 else: endpoints[endpoint] = endpoints[endpoint] + 1 client.close() retry += 1 wait_before_test(1) print(f"Retry #{retry}") assert len(endpoints) is 1 # Step 3 - restore to default load balancing method and confirm requests are balanced. self.restore_ts(kube_apis, transport_server_setup) wait_before_test() endpoints = {} retry = 0 while (len(endpoints) is not 3 and retry <= 30): for i in range(20): host = host.strip("[]") client = socket.create_connection((host, port)) client.sendall(b'connect') response = client.recv(4096) endpoint = response.decode() print(f' req number {i}; response: {endpoint}') if endpoint not in endpoints: endpoints[endpoint] = 1 else: endpoints[endpoint] = endpoints[endpoint] + 1 client.close() retry += 1 wait_before_test(1) print(f"Retry #{retry}") assert len(endpoints) is 3
def test_tcp_request_max_connections(self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites): """ The config, maxConns, should limit the number of open TCP connections. 3 replicas of max 2 connections is 6, so making the 7th connection will fail. """ # step 1 - set max connections to 2 with 1 replica patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/max-connections-transport-server.yaml" patch_ts_from_yaml( kube_apis.custom_objects, transport_server_setup.name, patch_src, transport_server_setup.namespace, ) wait_before_test() configs = 0 retry = 0 while (configs is not 3 and retry <= 30): result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace) pattern = 'max_conns=2' configs = len(re.findall(pattern, result_conf)) retry += 1 wait_before_test(1) print(f"Retry #{retry}") assert configs is 3 # step 2 - make the number of allowed connections port = transport_server_setup.public_endpoint.tcp_server_port host = transport_server_setup.public_endpoint.public_ip clients = [] for i in range(6): c = self.make_holding_connection(host, port) clients.append(c) # step 3 - assert the next connection fails try: c = self.make_holding_connection(host, port) # making a connection should fail and throw an exception assert c is None except ConnectionResetError as E: print("The expected exception occurred:", E) for c in clients: c.close() # step 4 - revert to config with no max connections patch_src = f"{TEST_DATA}/transport-server-tcp-load-balance/standard/transport-server.yaml" patch_ts_from_yaml( kube_apis.custom_objects, transport_server_setup.name, patch_src, transport_server_setup.namespace, ) wait_before_test() # step 5 - confirm making lots of connections doesn't cause an error clients = [] for i in range(24): c = self.make_holding_connection(host, port) clients.append(c) for c in clients: c.close()
def test_udp_passing_healthcheck_with_match( self, kube_apis, crd_ingress_controller, transport_server_setup, ingress_controller_prerequisites ): """ Configure a passing health check and check that all backend pods return responses. """ # Step 1 - configure a passing health check patch_src = f"{TEST_DATA}/transport-server-udp-load-balance/passing-hc-transport-server.yaml" patch_ts_from_yaml( kube_apis.custom_objects, transport_server_setup.name, patch_src, transport_server_setup.namespace, ) # 4s includes 3s timeout for a health check to fail in case a backend pod doesn't respond or responds with # an unexpected response wait_before_test(4) result_conf = get_ts_nginx_template_conf( kube_apis.v1, transport_server_setup.namespace, transport_server_setup.name, transport_server_setup.ingress_pod_name, ingress_controller_prerequisites.namespace ) match = f"match_ts_{transport_server_setup.namespace}_transport-server_udp-app" assert "health_check interval=5s" in result_conf assert f"passes=1 jitter=0s fails=1 udp match={match}" in result_conf assert "health_check_timeout 3s;" assert 'send "health"' in result_conf assert 'expect "healthy"' in result_conf # Step 2 - confirm load balancing works port = transport_server_setup.public_endpoint.udp_server_port host = transport_server_setup.public_endpoint.public_ip print(f"sending udp requests to: {host}:{port}") retry = 0 endpoints = {} while(len(endpoints) is not 3 and retry <= 30): for i in range(20): host = host.strip("[]") client = socket.socket(ipfamily_from_host(host), socket.SOCK_DGRAM, 0) client.sendto("ping".encode('utf-8'), (host, port)) data, address = client.recvfrom(4096) endpoint = data.decode() print(f' req number {i}; response: {endpoint}') if endpoint not in endpoints: endpoints[endpoint] = 1 else: endpoints[endpoint] = endpoints[endpoint] + 1 client.close() retry += 1 wait_before_test(1) print(f"Retry #{retry}") assert len(endpoints) is 3 # Step 3 - restore self.restore_ts(kube_apis, transport_server_setup)