def test_ap_multi_sec_logs(self, request, kube_apis,
crd_ingress_controller_with_ap,
appprotect_setup, test_namespace):
"""
Test corresponding log entries with multiple log destinations (in this case, two syslog servers)
"""
src_syslog2_yaml = f"{TEST_DATA}/appprotect/syslog2.yaml"
log_loc = "/var/log/messages"
print("Create a second syslog server")
create_items_from_yaml(kube_apis, src_syslog2_yaml, test_namespace)
syslog_dst = f"syslog-svc.{test_namespace}"
syslog2_dst = f"syslog2-svc.{test_namespace}"
syslog_pod = get_pod_name_that_contains(kube_apis.v1, test_namespace,
"syslog-")
syslog2_pod = get_pod_name_that_contains(kube_apis.v1, test_namespace,
"syslog2")
with open(src_ing_yaml) as f:
doc = yaml.safe_load(f)
doc["metadata"]["annotations"][
"appprotect.f5.com/app-protect-policy"] = ap_policy
doc["metadata"]["annotations"][
"appprotect.f5.com/app-protect-enable"] = "True"
doc["metadata"]["annotations"][
"appprotect.f5.com/app-protect-security-log-enable"] = "True"
# both lists need to be the same length, if one of the referenced configs is invalid/non-existent then no logconfs are applied.
doc["metadata"]["annotations"][
"appprotect.f5.com/app-protect-security-log"] = f"{test_namespace}/logconf,{test_namespace}/logconf"
doc["metadata"]["annotations"][
"appprotect.f5.com/app-protect-security-log-destination"] = f"syslog:server={syslog_dst}:514,syslog:server={syslog2_dst}:514"
create_ingress(kube_apis.networking_v1, test_namespace, doc)
ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
wait_before_test(30)
ensure_response_from_backend(appprotect_setup.req_url,
ingress_host,
check404=True)
print("----------------------- Send request ----------------------")
response = requests.get(appprotect_setup.req_url + "/<script>",
headers={"host": ingress_host},
verify=False)
print(response.text)
log_contents = ""
log2_contents = ""
retry = 0
while ("ASM:attack_type" not in log_contents
and "ASM:attack_type" not in log2_contents and retry <= 60):
log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod,
test_namespace)
log2_contents = get_file_contents(kube_apis.v1, log_loc,
syslog2_pod, test_namespace)
retry += 1
wait_before_test(1)
print(f"Security log not updated, retrying... #{retry}")
reload_ms = get_last_reload_time(appprotect_setup.metrics_url, "nginx")
print(f"last reload duration: {reload_ms} ms")
reload_times[
f"{request.node.name}"] = f"last reload duration: {reload_ms} ms"
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
delete_items_from_yaml(kube_apis, src_syslog2_yaml, test_namespace)
clear_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
assert_invalid_responses(response)
# check logs in dest. #1 i.e. syslog server #1
assert (
'ASM:attack_type="Non-browser Client,Abuse of Functionality,Cross Site Scripting (XSS)"'
in log_contents and 'severity="Critical"' in log_contents
and 'request_status="blocked"' in log_contents
and 'outcome="REJECTED"' in log_contents)
# check logs in dest. #2 i.e. syslog server #2
assert (
'ASM:attack_type="Non-browser Client,Abuse of Functionality,Cross Site Scripting (XSS)"'
in log2_contents and 'severity="Critical"' in log2_contents
and 'request_status="blocked"' in log2_contents
and 'outcome="REJECTED"' in log2_contents)
def test_ap_sec_logs_on(
self,
request,
kube_apis,
ingress_controller_prerequisites,
crd_ingress_controller_with_ap,
appprotect_setup,
test_namespace,
):
"""
Test corresponding log entries with correct policy (includes setting up a syslog server as defined in syslog.yaml)
"""
log_loc = "/var/log/messages"
syslog_dst = f"syslog-svc.{test_namespace}"
syslog_pod = get_pod_name_that_contains(kube_apis.v1, test_namespace,
"syslog-")
create_ingress_with_ap_annotations(kube_apis, src_ing_yaml,
test_namespace, ap_policy, "True",
"True", f"{syslog_dst}:514")
ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
print(
"--------- Run test while AppProtect module is enabled with correct policy ---------"
)
ensure_response_from_backend(appprotect_setup.req_url,
ingress_host,
check404=True)
print(
"----------------------- Send invalid request ----------------------"
)
response_block = requests.get(appprotect_setup.req_url + "/<script>",
headers={"host": ingress_host},
verify=False)
print(response_block.text)
log_contents_block = ""
retry = 0
while "ASM:attack_type" not in log_contents_block and retry <= 30:
log_contents_block = get_file_contents(kube_apis.v1, log_loc,
syslog_pod, test_namespace)
retry += 1
wait_before_test(1)
print(f"Security log not updated, retrying... #{retry}")
print(
"----------------------- Send valid request ----------------------"
)
headers = {
"Host":
ingress_host,
"User-Agent":
"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0",
}
response = requests.get(appprotect_setup.req_url,
headers=headers,
verify=False)
print(response.text)
wait_before_test(10)
log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod,
test_namespace)
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
clear_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
assert_invalid_responses(response_block)
assert (
'ASM:attack_type="Non-browser Client,Abuse of Functionality,Cross Site Scripting (XSS)"'
in log_contents_block)
assert 'severity="Critical"' in log_contents_block
assert 'request_status="blocked"' in log_contents_block
assert 'outcome="REJECTED"' in log_contents_block
assert_valid_responses(response)
assert 'ASM:attack_type="N/A"' in log_contents
assert 'severity="Informational"' in log_contents
assert 'request_status="passed"' in log_contents
assert 'outcome="PASSED"' in log_contents
def test_dos_arbitrator(self, kube_apis, ingress_controller_prerequisites,
crd_ingress_controller_with_dos, dos_setup,
test_namespace):
"""
Test App Protect Dos: Check new IC pod get learning info
"""
print(
"----------------------- Get syslog pod name ----------------------"
)
syslog_pod = self.getPodNameThatContains(
kube_apis, ingress_controller_prerequisites.namespace, "syslog")
assert "syslog" in syslog_pod
log_loc = f"/var/log/messages"
clear_file_contents(kube_apis.v1, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace)
print(
"------------------------- Deploy ingress -----------------------------"
)
create_ingress_with_dos_annotations(kube_apis, src_ing_yaml,
test_namespace,
test_namespace + "/dos-protected")
ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
# print("------------------------- Learning Phase -----------------------------")
print("start good clients requests")
p_good_client = subprocess.Popen([
f"exec {TEST_DATA}/dos/good_clients_xff.sh {ingress_host} {dos_setup.req_url}"
],
preexec_fn=os.setsid,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
print("Learning for max 10 minutes")
find_in_log(kube_apis, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace, 600,
"learning_confidence=\"Ready\"")
print(
"------------------------- Check new IC pod get info from arbitrator -----------------------------"
)
ic_ns = ingress_controller_prerequisites.namespace
scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress",
ic_ns, 2)
while get_pods_amount_with_name(kube_apis.v1, "nginx-ingress",
"nginx-ingress") is not 2:
print(f"Number of replicas is not 2, retrying...")
wait_before_test()
print(
"------------------------- Check if new pod receive info from arbitrator -----------------------------"
)
print("Wait for 30 seconds")
wait_before_test(30)
log_contents = get_file_contents(
kube_apis.v1, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace)
log_info_dic = log_content_to_dic(log_contents)
print("Stop Good Client")
p_good_client.terminate()
learning_units_hostname = []
for log in log_info_dic:
if log['unit_hostname'] not in learning_units_hostname and log[
'learning_confidence'] == "Ready":
learning_units_hostname.append(log['unit_hostname'])
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
assert (len(learning_units_hostname) == 2)
def test_dos_under_attack_with_learning(self, kube_apis,
ingress_controller_prerequisites,
crd_ingress_controller_with_dos,
dos_setup, test_namespace):
"""
Test App Protect Dos: Block bad clients attack with learning
"""
log_loc = f"/var/log/messages"
print(
"----------------------- Get syslog pod name ----------------------"
)
syslog_pod = self.getPodNameThatContains(
kube_apis, ingress_controller_prerequisites.namespace, "syslog")
assert "syslog" in syslog_pod
clear_file_contents(kube_apis.v1, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace)
print(
"------------------------- Deploy ingress -----------------------------"
)
create_ingress_with_dos_annotations(kube_apis, src_ing_yaml,
test_namespace,
test_namespace + "/dos-protected")
ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
print(
"------------------------- Learning Phase -----------------------------"
)
print("start good clients requests")
p_good_client = subprocess.Popen([
f"exec {TEST_DATA}/dos/good_clients_xff.sh {ingress_host} {dos_setup.req_url}"
],
preexec_fn=os.setsid,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
print("Learning for max 10 minutes")
find_in_log(kube_apis, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace, 600,
"learning_confidence=\"Ready\"")
print("------------------------- Attack -----------------------------")
print("start bad clients requests")
p_attack = subprocess.Popen([
f"exec {TEST_DATA}/dos/bad_clients_xff.sh {ingress_host} {dos_setup.req_url}"
],
preexec_fn=os.setsid,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
print("Attack for 300 seconds")
wait_before_test(300)
print("Stop Attack")
p_attack.terminate()
print("wait max 140 seconds after attack stop, to get attack ended")
find_in_log(kube_apis, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace, 140,
"attack_event=\"Attack ended\"")
print("Stop Good Client")
p_good_client.terminate()
log_contents = get_file_contents(
kube_apis.v1, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace)
log_info_dic = log_content_to_dic(log_contents)
# Analyze the log
no_attack = False
attack_started = False
under_attack = False
attack_ended = False
bad_actor_detected = False
signature_detected = False
health_ok = False
bad_ip = ['1.1.1.1', '1.1.1.2', '1.1.1.3']
fmt = '%b %d %Y %H:%M:%S'
for log in log_info_dic:
if log['attack_event'] == 'No Attack':
if int(log['dos_attack_id']) == 0 and not no_attack:
no_attack = True
elif log['attack_event'] == 'Attack started':
if int(log['dos_attack_id']) > 0 and not attack_started:
attack_started = True
start_attack_time = datetime.strptime(
log['date_time'], fmt)
elif log['attack_event'] == 'Under Attack':
under_attack = True
if not health_ok and float(log['stress_level']) < 0.6:
health_ok = True
health_ok_time = datetime.strptime(log['date_time'], fmt)
elif log['attack_event'] == 'Attack signature detected':
signature_detected = True
elif log['attack_event'] == 'Bad actors detected':
if under_attack:
bad_actor_detected = True
elif log['attack_event'] == 'Bad actor detection':
if under_attack and log['source_ip'] in bad_ip:
bad_ip.remove(log['source_ip'])
elif log['attack_event'] == 'Attack ended':
attack_ended = True
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
assert (no_attack and attack_started and under_attack and attack_ended
and health_ok
and (health_ok_time - start_attack_time).total_seconds() < 150
and signature_detected and bad_actor_detected
and len(bad_ip) == 0)
def test_dos_under_attack_no_learning(self, kube_apis,
ingress_controller_prerequisites,
crd_ingress_controller_with_dos,
dos_setup, test_namespace):
"""
Test App Protect Dos: Block bad clients attack
"""
log_loc = f"/var/log/messages"
print(
"----------------------- Get syslog pod name ----------------------"
)
syslog_pod = self.getPodNameThatContains(
kube_apis, ingress_controller_prerequisites.namespace, "syslog")
assert "syslog" in syslog_pod
clear_file_contents(kube_apis.v1, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace)
print(
"------------------------- Deploy ingress -----------------------------"
)
create_ingress_with_dos_annotations(kube_apis, src_ing_yaml,
test_namespace,
test_namespace + "/dos-protected")
ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
print("------------------------- Attack -----------------------------")
wait_before_test(10)
print("start bad clients requests")
p_attack = subprocess.Popen([
f"exec {TEST_DATA}/dos/bad_clients_xff.sh {ingress_host} {dos_setup.req_url}"
],
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
print("Attack for 30 seconds")
wait_before_test(30)
print("Stop Attack")
p_attack.terminate()
print("wait max 140 seconds after attack stop, to get attack ended")
find_in_log(kube_apis, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace, 140,
"attack_event=\"Attack ended\"")
log_contents = get_file_contents(
kube_apis.v1, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace)
log_info_dic = log_content_to_dic(log_contents)
# Analyze the log
no_attack = False
attack_started = False
under_attack = False
attack_ended = False
for log in log_info_dic:
# Start with no attack
if log['attack_event'] == "No Attack" and int(
log['dos_attack_id']) == 0 and not no_attack:
no_attack = True
# Attack started
elif log['attack_event'] == "Attack started" and int(
log['dos_attack_id']) > 0 and not attack_started:
attack_started = True
# Under attack
elif log['attack_event'] == "Under Attack" and int(
log['dos_attack_id']) > 0 and not under_attack:
under_attack = True
# Attack ended
elif log['attack_event'] == "Attack ended" and int(
log['dos_attack_id']) > 0 and not attack_ended:
attack_ended = True
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
assert (no_attack and attack_started and under_attack and attack_ended)
def test_dos_sec_logs_on(
self,
kube_apis,
ingress_controller_prerequisites,
crd_ingress_controller_with_dos,
dos_setup,
test_namespace,
):
"""
Test corresponding log entries with correct policy (includes setting up a syslog server as defined in syslog.yaml)
"""
print(
"----------------------- Get syslog pod name ----------------------"
)
syslog_pod = self.getPodNameThatContains(
kube_apis, ingress_controller_prerequisites.namespace, "syslog")
assert "syslog" in syslog_pod
log_loc = f"/var/log/messages"
clear_file_contents(kube_apis.v1, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace)
create_ingress_with_dos_annotations(kube_apis, src_ing_yaml,
test_namespace,
test_namespace + "/dos-protected")
ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
print(
"--------- Run test while DOS module is enabled with correct policy ---------"
)
ensure_response_from_backend(dos_setup.req_url,
ingress_host,
check404=True)
pod_name = self.getPodNameThatContains(
kube_apis, ingress_controller_prerequisites.namespace,
"nginx-ingress")
get_ingress_nginx_template_conf(kube_apis.v1, test_namespace,
"dos-ingress", pod_name,
"nginx-ingress")
print("----------------------- Send request ----------------------")
response = requests.get(dos_setup.req_url,
headers={"host": "dos.example.com"},
verify=False)
print(response.text)
wait_before_test(10)
print(
f'log_loc {log_loc} syslog_pod {syslog_pod} namespace {ingress_controller_prerequisites.namespace}'
)
log_contents = get_file_contents(
kube_apis.v1, log_loc, syslog_pod,
ingress_controller_prerequisites.namespace)
delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
print(log_contents)
assert 'product="app-protect-dos"' in log_contents
assert f'vs_name="{test_namespace}/dos-protected/name"' in log_contents
assert 'bad_actor' in log_contents