def test_startup_time(self, request, kube_apis,
ingress_controller_prerequisites,
crd_ingress_controller, ingress_controller_endpoint,
vs_vsr_setup):
"""
Pod startup time with 1 VS and multiple VSRs.
"""
total_vsr = int(request.config.getoption("--batch-resources"))
ic_ns = ingress_controller_prerequisites.namespace
scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress",
ic_ns, 0)
while get_pods_amount(kube_apis.v1, ic_ns) is not 0:
print(f"Number of replicas not 0, retrying...")
wait_before_test()
num = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api,
"nginx-ingress", ic_ns, 1)
metrics_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics"
assert (get_total_vs(metrics_url, "nginx") == "1"
and get_total_vsr(metrics_url, "nginx") == str(total_vsr)
and get_last_reload_status(metrics_url, "nginx") == "1")
assert num is None
def test_ap_ingress_batch_start(
self,
request,
kube_apis,
crd_ingress_controller_with_ap,
ap_ingress_setup,
ingress_controller_prerequisites,
test_namespace,
):
"""
Pod startup time with AP Ingress
"""
print(
"------------- Run test for AP policy: dataguard-alarm --------------"
)
print(
f"Request URL: {ap_ingress_setup.req_url} and Host: {ap_ingress_setup.ingress_host}"
)
ensure_response_from_backend(ap_ingress_setup.req_url,
ap_ingress_setup.ingress_host,
check404=True)
total_ing = int(request.config.getoption("--batch-resources"))
manifest = f"{TEST_DATA}/appprotect/appprotect-ingress.yaml"
for i in range(1, total_ing + 1):
with open(manifest) as f:
doc = yaml.safe_load(f)
doc["metadata"]["name"] = f"appprotect-ingress-{i}"
doc["spec"]["rules"][0]["host"] = f"appprotect-{i}.example.com"
create_ingress(kube_apis.networking_v1, test_namespace, doc)
print(f"Total resources deployed is {total_ing}")
wait_before_test()
ic_ns = ingress_controller_prerequisites.namespace
scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress",
ic_ns, 0)
while get_pods_amount(kube_apis.v1, ic_ns) is not 0:
print(f"Number of replicas not 0, retrying...")
wait_before_test()
num = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api,
"nginx-ingress", ic_ns, 1)
assert (get_total_ingresses(ap_ingress_setup.metrics_url, "nginx")
== str(total_ing + 1) and get_last_reload_status(
ap_ingress_setup.metrics_url, "nginx") == "1")
for i in range(1, total_ing + 1):
delete_ingress(kube_apis.networking_v1, f"appprotect-ingress-{i}",
test_namespace)
assert num is None
def test_vs_batch_start(
self,
request,
kube_apis,
ingress_controller_prerequisites,
crd_ingress_controller,
virtual_server_setup,
test_namespace,
):
"""
Pod startup time with simple VS
"""
resp = requests.get(virtual_server_setup.backend_1_url,
headers={"host": virtual_server_setup.vs_host})
assert resp.status_code is 200
total_vs = int(request.config.getoption("--batch-resources"))
manifest = f"{TEST_DATA}/virtual-server/standard/virtual-server.yaml"
for i in range(1, total_vs + 1):
with open(manifest) as f:
doc = yaml.safe_load(f)
doc["metadata"]["name"] = f"virtual-server-{i}"
doc["spec"]["host"] = f"virtual-server-{i}.example.com"
kube_apis.custom_objects.create_namespaced_custom_object(
"k8s.nginx.org", "v1", test_namespace, "virtualservers",
doc)
print(
f"VirtualServer created with name '{doc['metadata']['name']}'"
)
print(f"Total resources deployed is {total_vs}")
wait_before_test()
ic_ns = ingress_controller_prerequisites.namespace
scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress",
ic_ns, 0)
while get_pods_amount(kube_apis.v1, ic_ns) is not 0:
print(f"Number of replicas not 0, retrying...")
wait_before_test()
num = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api,
"nginx-ingress", ic_ns, 1)
assert (get_total_vs(virtual_server_setup.metrics_url, "nginx")
== str(total_vs + 1) and get_last_reload_status(
virtual_server_setup.metrics_url, "nginx") == "1")
for i in range(1, total_vs + 1):
delete_virtual_server(kube_apis.custom_objects,
f"virtual-server-{i}", test_namespace)
assert num is None
def test_reload_count_after_start(self, kube_apis, smoke_setup,
ingress_controller_prerequisites):
ns = ingress_controller_prerequisites.namespace
scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress",
ns, 0)
while get_pods_amount(kube_apis.v1, ns) is not 0:
print(f"Number of replicas not 0, retrying...")
wait_before_test()
num = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api,
"nginx-ingress", ns, 1)
assert num is None
metrics_url = f"http://{smoke_setup.public_endpoint.public_ip}:{smoke_setup.public_endpoint.metrics_port}/metrics"
count = get_reload_count(metrics_url)
assert count == 1
def test_ap_pod_startup(
self,
request,
kube_apis,
ingress_controller_prerequisites,
crd_ingress_controller_with_ap,
appprotect_setup,
test_namespace,
):
"""
Log pod startup time while scaling up from 0 to 1
"""
src_syslog_yaml = f"{TEST_DATA}/appprotect/syslog.yaml"
create_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
syslog_ep = get_service_endpoint(kube_apis, "syslog-svc",
test_namespace)
# items[-1] because syslog pod is last one to spin-up
syslog_pod = kube_apis.v1.list_namespaced_pod(
test_namespace).items[-1].metadata.name
create_ingress_with_ap_annotations(kube_apis, src_ing_yaml,
test_namespace, ap_policy, "True",
"True", f"{syslog_ep}:514")
ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
print(
"--------- AppProtect module is enabled with correct policy ---------"
)
ensure_response_from_backend(appprotect_setup.req_url,
ingress_host,
check404=True)
ns = ingress_controller_prerequisites.namespace
scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress",
ns, 0)
while get_pods_amount(kube_apis.v1, ns) is not 0:
print(f"Number of replicas not 0, retrying...")
wait_before_test()
num = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api,
"nginx-ingress", ns, 1)
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
assert num is None
def test_ssl_keys(self, cli_arguments, kube_apis,
ingress_controller_prerequisites, crd_ingress_controller,
virtual_server_setup, clean_up):
ic_pods_amount = get_pods_amount(
kube_apis.v1, ingress_controller_prerequisites.namespace)
ic_pod_name = get_first_pod_name(
kube_apis.v1, ingress_controller_prerequisites.namespace)
initial_list = get_events(kube_apis.v1, virtual_server_setup.namespace)
print("Step 1: update ConfigMap with valid ssl keys")
replace_configmap_from_yaml(
kube_apis.v1,
ingress_controller_prerequisites.config_map['metadata']['name'],
ingress_controller_prerequisites.namespace,
f"{TEST_DATA}/virtual-server-configmap-keys/configmap-ssl-keys.yaml"
)
wait_before_test(1)
step_1_events = get_events(kube_apis.v1,
virtual_server_setup.namespace)
step_1_config = get_vs_nginx_template_conf(
kube_apis.v1, virtual_server_setup.namespace,
virtual_server_setup.vs_name, ic_pod_name,
ingress_controller_prerequisites.namespace)
assert_update_events_emitted(virtual_server_setup, step_1_events,
initial_list, ic_pods_amount)
assert_ssl_keys(step_1_config)
print("Step 2: update ConfigMap with invalid ssl keys")
replace_configmap_from_yaml(
kube_apis.v1,
ingress_controller_prerequisites.config_map['metadata']['name'],
ingress_controller_prerequisites.namespace,
f"{TEST_DATA}/virtual-server-configmap-keys/configmap-ssl-keys-invalid.yaml"
)
wait_before_test(1)
step_2_events = get_events(kube_apis.v1,
virtual_server_setup.namespace)
step_2_config = get_vs_nginx_template_conf(
kube_apis.v1, virtual_server_setup.namespace,
virtual_server_setup.vs_name, ic_pod_name,
ingress_controller_prerequisites.namespace)
assert_update_event_count_increased(virtual_server_setup,
step_2_events, step_1_events)
assert_defaults_of_ssl_keys(step_2_config)
def test_ap_pod_startup(
self,
request,
kube_apis,
ingress_controller_prerequisites,
crd_ingress_controller_with_ap,
appprotect_setup,
test_namespace,
):
"""
Log pod startup time while scaling up from 0 to 1
"""
syslog_dst = f"syslog-svc.{test_namespace}"
create_ingress_with_ap_annotations(kube_apis, src_ing_yaml,
test_namespace, ap_policy, "True",
"True", f"{syslog_dst}:514")
ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
print(
"--------- AppProtect module is enabled with correct policy ---------"
)
ensure_response_from_backend(appprotect_setup.req_url,
ingress_host,
check404=True)
ns = ingress_controller_prerequisites.namespace
scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress",
ns, 0)
while get_pods_amount(kube_apis.v1, ns) is not 0:
print(f"Number of replicas not 0, retrying...")
wait_before_test()
num = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api,
"nginx-ingress", ns, 1)
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
assert num is None
def test_keys_in_main_config(self, cli_arguments, kube_apis,
ingress_controller_prerequisites,
crd_ingress_controller, virtual_server_setup,
clean_up):
wait_before_test(1)
ic_pods_amount = get_pods_amount(
kube_apis.v1, ingress_controller_prerequisites.namespace)
ic_pod_name = get_first_pod_name(
kube_apis.v1, ingress_controller_prerequisites.namespace)
initial_list = get_events(kube_apis.v1, virtual_server_setup.namespace)
data_file = f"{TEST_DATA}/virtual-server-configmap-keys/configmap-validation-keys.yaml"
data_file_invalid = f"{TEST_DATA}/virtual-server-configmap-keys/configmap-validation-keys-invalid.yaml"
config_path = "/etc/nginx/nginx.conf"
print(
"Step 5: main config: update ConfigMap with valid keys with validation rules"
)
replace_configmap_from_yaml(
kube_apis.v1,
ingress_controller_prerequisites.config_map['metadata']['name'],
ingress_controller_prerequisites.namespace, data_file)
expected_values = get_configmap_fields_from_yaml(data_file)
wait_before_test(1)
step_5_config = get_file_contents(
kube_apis.v1, config_path, ic_pod_name,
ingress_controller_prerequisites.namespace)
step_5_events = get_events(kube_apis.v1,
virtual_server_setup.namespace)
assert_update_event_count_increased(virtual_server_setup,
step_5_events, initial_list)
assert_keys_with_validation_in_main_config(step_5_config,
expected_values)
print("Step 6: main config: update ConfigMap with invalid keys")
replace_configmap_from_yaml(
kube_apis.v1,
ingress_controller_prerequisites.config_map['metadata']['name'],
ingress_controller_prerequisites.namespace, data_file_invalid)
unexpected_values = get_configmap_fields_from_yaml(data_file_invalid)
wait_before_test(1)
step_6_config = get_file_contents(
kube_apis.v1, config_path, ic_pod_name,
ingress_controller_prerequisites.namespace)
step_6_events = get_events(kube_apis.v1,
virtual_server_setup.namespace)
assert_update_event_count_increased(virtual_server_setup,
step_6_events, step_5_events)
assert_defaults_of_keys_with_validation_in_main_config(
step_6_config, unexpected_values)
print("Step 7: main config: special case for hash variables")
data_file = f"{TEST_DATA}/virtual-server-configmap-keys/configmap-global-variables.yaml"
expected_values = get_configmap_fields_from_yaml(data_file)
replace_configmap_from_yaml(
kube_apis.v1,
ingress_controller_prerequisites.config_map['metadata']['name'],
ingress_controller_prerequisites.namespace, data_file)
wait_before_test(1)
step_7_config = get_file_contents(
kube_apis.v1, config_path, ic_pod_name,
ingress_controller_prerequisites.namespace)
step_7_events = get_events(kube_apis.v1,
virtual_server_setup.namespace)
assert_not_applied_events_emitted(virtual_server_setup, step_7_events,
step_6_events, ic_pods_amount)
assert_keys_with_validation_in_main_config(step_7_config,
expected_values)
def test_keys(self, cli_arguments, kube_apis,
ingress_controller_prerequisites, crd_ingress_controller,
virtual_server_setup, clean_up):
ic_pods_amount = get_pods_amount(
kube_apis.v1, ingress_controller_prerequisites.namespace)
ic_pod_name = get_first_pod_name(
kube_apis.v1, ingress_controller_prerequisites.namespace)
initial_list = get_events(kube_apis.v1, virtual_server_setup.namespace)
print(
"Step 1: update ConfigMap with valid keys without validation rules"
)
replace_configmap_from_yaml(
kube_apis.v1,
ingress_controller_prerequisites.config_map['metadata']['name'],
ingress_controller_prerequisites.namespace,
f"{TEST_DATA}/virtual-server-configmap-keys/configmap-no-validation-keys.yaml"
)
expected_values = get_configmap_fields_from_yaml(
f"{TEST_DATA}/virtual-server-configmap-keys/configmap-no-validation-keys.yaml"
)
wait_before_test(1)
step_1_events = get_events(kube_apis.v1,
virtual_server_setup.namespace)
step_1_config = get_vs_nginx_template_conf(
kube_apis.v1, virtual_server_setup.namespace,
virtual_server_setup.vs_name, ic_pod_name,
ingress_controller_prerequisites.namespace)
assert_update_events_emitted(virtual_server_setup, step_1_events,
initial_list, ic_pods_amount)
assert_keys_without_validation(step_1_config, expected_values)
print(
"Step 2: update ConfigMap with invalid keys without validation rules"
)
cm_src = f"{TEST_DATA}/virtual-server-configmap-keys/configmap-no-validation-keys-invalid.yaml"
replace_configmap_from_yaml(
kube_apis.v1,
ingress_controller_prerequisites.config_map['metadata']['name'],
ingress_controller_prerequisites.namespace, cm_src)
expected_values = get_configmap_fields_from_yaml(
f"{TEST_DATA}/virtual-server-configmap-keys/configmap-no-validation-keys-invalid.yaml"
)
wait_before_test(1)
step_2_events = get_events(kube_apis.v1,
virtual_server_setup.namespace)
step_2_config = get_vs_nginx_template_conf(
kube_apis.v1, virtual_server_setup.namespace,
virtual_server_setup.vs_name, ic_pod_name,
ingress_controller_prerequisites.namespace)
assert_not_applied_events_emitted(virtual_server_setup, step_2_events,
step_1_events, ic_pods_amount)
assert_keys_without_validation(step_2_config, expected_values)
# to cover the OSS case, this will be changed in the future
if cli_arguments['ic-type'] == "nginx-ingress":
data_file = f"{TEST_DATA}/virtual-server-configmap-keys/configmap-validation-keys-oss.yaml"
data_file_invalid = f"{TEST_DATA}/virtual-server-configmap-keys/configmap-validation-keys-invalid-oss.yaml"
else:
data_file = f"{TEST_DATA}/virtual-server-configmap-keys/configmap-validation-keys.yaml"
data_file_invalid = f"{TEST_DATA}/virtual-server-configmap-keys/configmap-validation-keys-invalid.yaml"
print("Step 3: update ConfigMap with valid keys with validation rules")
replace_configmap_from_yaml(
kube_apis.v1,
ingress_controller_prerequisites.config_map['metadata']['name'],
ingress_controller_prerequisites.namespace, data_file)
expected_values = get_configmap_fields_from_yaml(data_file)
wait_before_test(1)
step_3_config = get_vs_nginx_template_conf(
kube_apis.v1, virtual_server_setup.namespace,
virtual_server_setup.vs_name, ic_pod_name,
ingress_controller_prerequisites.namespace)
step_3_events = get_events(kube_apis.v1,
virtual_server_setup.namespace)
assert_update_event_count_increased(virtual_server_setup,
step_3_events, step_2_events)
assert_keys_with_validation(step_3_config, expected_values)
# to cover the OSS case, this will be changed in the future
if cli_arguments['ic-type'] == "nginx-ingress":
assert_specific_keys_for_nginx_oss(step_3_config, expected_values)
else:
assert_specific_keys_for_nginx_plus(step_3_config, expected_values)
print("Step 4: update ConfigMap with invalid keys")
replace_configmap_from_yaml(
kube_apis.v1,
ingress_controller_prerequisites.config_map['metadata']['name'],
ingress_controller_prerequisites.namespace, data_file_invalid)
expected_values = get_configmap_fields_from_yaml(data_file_invalid)
wait_before_test(1)
step_4_config = get_vs_nginx_template_conf(
kube_apis.v1, virtual_server_setup.namespace,
virtual_server_setup.vs_name, ic_pod_name,
ingress_controller_prerequisites.namespace)
step_4_events = get_events(kube_apis.v1,
virtual_server_setup.namespace)
assert_update_event_count_increased(virtual_server_setup,
step_4_events, step_3_events)
assert_defaults_of_keys_with_validation(step_4_config, expected_values)
def test_virtual_server_behavior(self,
kube_apis, cli_arguments,
ingress_controller_prerequisites,
crd_ingress_controller,
virtual_server_setup):
ic_pods_amount = get_pods_amount(kube_apis.v1, ingress_controller_prerequisites.namespace)
ic_pod_name = get_first_pod_name(kube_apis.v1, ingress_controller_prerequisites.namespace)
print("Step 1: initial check")
step_1_list = get_events(kube_apis.v1, virtual_server_setup.namespace)
assert_vs_conf_exists(kube_apis, ic_pod_name, ingress_controller_prerequisites.namespace,
virtual_server_setup)
assert_response_200(virtual_server_setup)
print("Step 2: make a valid VirtualServer invalid and check")
patch_virtual_server_from_yaml(kube_apis.custom_objects,
virtual_server_setup.vs_name,
f"{TEST_DATA}/virtual-server-validation/virtual-server-invalid-cookie.yaml",
virtual_server_setup.namespace)
wait_before_test(1)
step_2_list = get_events(kube_apis.v1, virtual_server_setup.namespace)
assert_reject_events_emitted(virtual_server_setup, step_2_list, step_1_list, ic_pods_amount)
assert_vs_conf_not_exists(kube_apis, ic_pod_name, ingress_controller_prerequisites.namespace,
virtual_server_setup)
assert_response_404(virtual_server_setup)
print("Step 3: update an invalid VirtualServer with another invalid and check")
patch_virtual_server_from_yaml(kube_apis.custom_objects,
virtual_server_setup.vs_name,
f"{TEST_DATA}/virtual-server-validation/virtual-server-no-default-action.yaml",
virtual_server_setup.namespace)
wait_before_test(1)
step_3_list = get_events(kube_apis.v1, virtual_server_setup.namespace)
assert_reject_events_emitted(virtual_server_setup, step_3_list, step_2_list, ic_pods_amount)
assert_vs_conf_not_exists(kube_apis, ic_pod_name, ingress_controller_prerequisites.namespace,
virtual_server_setup)
assert_response_404(virtual_server_setup)
print("Step 4: make an invalid VirtualServer valid and check")
patch_virtual_server_from_yaml(kube_apis.custom_objects,
virtual_server_setup.vs_name,
f"{TEST_DATA}/virtual-server-validation/standard/virtual-server.yaml",
virtual_server_setup.namespace)
wait_before_test(1)
step_4_list = get_events(kube_apis.v1, virtual_server_setup.namespace)
assert_vs_conf_exists(kube_apis, ic_pod_name, ingress_controller_prerequisites.namespace,
virtual_server_setup)
assert_event_count_increased_in_list(virtual_server_setup, step_4_list, step_3_list)
assert_response_200(virtual_server_setup)
print("Step 5: delete VS and then create an invalid and check")
delete_virtual_server(kube_apis.custom_objects, virtual_server_setup.vs_name, virtual_server_setup.namespace)
create_virtual_server_from_yaml(kube_apis.custom_objects,
f"{TEST_DATA}/virtual-server-validation/virtual-server-invalid-cookie.yaml",
virtual_server_setup.namespace)
wait_before_test(1)
step_5_list = get_events(kube_apis.v1, virtual_server_setup.namespace)
assert_reject_events_emitted(virtual_server_setup, step_5_list, step_4_list, ic_pods_amount)
assert_vs_conf_not_exists(kube_apis, ic_pod_name, ingress_controller_prerequisites.namespace,
virtual_server_setup)
assert_response_404(virtual_server_setup)
def test_ap_waf_policy_vs_batch_start(
self,
request,
kube_apis,
ingress_controller_prerequisites,
crd_ingress_controller_with_ap,
virtual_server_setup,
appprotect_waf_setup,
test_namespace,
):
"""
Pod startup time with AP WAF Policy
"""
waf_spec_vs_src = f"{TEST_DATA}/ap-waf/virtual-server-waf-spec.yaml"
waf_pol_dataguard_src = f"{TEST_DATA}/ap-waf/policies/waf-dataguard.yaml"
print(f"Create waf policy")
create_ap_waf_policy_from_yaml(
kube_apis.custom_objects,
waf_pol_dataguard_src,
test_namespace,
test_namespace,
True,
False,
ap_pol_name,
log_name,
"syslog:server=127.0.0.1:514",
)
wait_before_test()
print(f"Patch vs with policy: {waf_spec_vs_src}")
patch_virtual_server_from_yaml(
kube_apis.custom_objects,
virtual_server_setup.vs_name,
waf_spec_vs_src,
virtual_server_setup.namespace,
)
wait_before_test(120)
print(
"----------------------- Send request with embedded malicious script----------------------"
)
response1 = requests.get(
virtual_server_setup.backend_1_url + "</script>",
headers={"host": virtual_server_setup.vs_host},
)
print(response1.status_code)
print(
"----------------------- Send request with blocked keyword in UDS----------------------"
)
response2 = requests.get(
virtual_server_setup.backend_1_url,
headers={"host": virtual_server_setup.vs_host},
data="kic",
)
total_vs = int(request.config.getoption("--batch-resources"))
print(response2.status_code)
for i in range(1, total_vs + 1):
with open(waf_spec_vs_src) as f:
doc = yaml.safe_load(f)
doc["metadata"]["name"] = f"virtual-server-{i}"
doc["spec"]["host"] = f"virtual-server-{i}.example.com"
kube_apis.custom_objects.create_namespaced_custom_object(
"k8s.nginx.org", "v1", test_namespace, "virtualservers",
doc)
print(
f"VirtualServer created with name '{doc['metadata']['name']}'"
)
print(f"Total resources deployed is {total_vs}")
wait_before_test()
ic_ns = ingress_controller_prerequisites.namespace
scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress",
ic_ns, 0)
while get_pods_amount(kube_apis.v1, ic_ns) is not 0:
print(f"Number of replicas not 0, retrying...")
wait_before_test()
num = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api,
"nginx-ingress", ic_ns, 1)
assert (get_total_vs(virtual_server_setup.metrics_url, "nginx")
== str(total_vs + 1) and get_last_reload_status(
virtual_server_setup.metrics_url, "nginx") == "1")
for i in range(1, total_vs + 1):
delete_virtual_server(kube_apis.custom_objects,
f"virtual-server-{i}", test_namespace)
delete_policy(kube_apis.custom_objects, "waf-policy", test_namespace)
assert num is None
