def container_killing_in_pod(cont_scenario):
scenario_name = cont_scenario.get("name", "")
namespace = cont_scenario.get("namespace", "*")
label_selector = cont_scenario.get("label_selector", None)
pod_names = cont_scenario.get("pod_names", [])
container_name = cont_scenario.get("container_name", "")
kill_action = cont_scenario.get("action", "kill 1")
kill_count = cont_scenario.get("count", 1)
if type(pod_names) != list:
logging.error("Please make sure your pod_names are in a list format")
sys.exit(1)
if len(pod_names) == 0:
if namespace == "*":
# returns double array of pod name and namespace
pods = kubecli.get_all_pods(label_selector)
else:
# Only returns pod names
pods = kubecli.list_pods(namespace, label_selector)
else:
if namespace == "*":
logging.error("You must specify the namespace to kill a container in a specific pod")
logging.error("Scenario " + scenario_name + " failed")
sys.exit(1)
pods = pod_names
# get container and pod name
container_pod_list = []
for pod in pods:
if type(pod) == list:
container_names = runcommand.invoke(
'kubectl get pods %s -n %s -o jsonpath="{.spec.containers[*].name}"' % (pod[0], pod[1])
).split(" ")
container_pod_list.append([pod[0], pod[1], container_names])
else:
container_names = runcommand.invoke(
'oc get pods %s -n %s -o jsonpath="{.spec.containers[*].name}"' % (pod, namespace)
).split(" ")
container_pod_list.append([pod, namespace, container_names])
killed_count = 0
killed_container_list = []
while killed_count < kill_count:
if len(container_pod_list) == 0:
logging.error("Trying to kill more containers than were found, try lowering kill count")
logging.error("Scenario " + scenario_name + " failed")
sys.exit(1)
selected_container_pod = container_pod_list[random.randint(0, len(container_pod_list) - 1)]
for c_name in selected_container_pod[2]:
if container_name != "":
if c_name == container_name:
killed_container_list.append([selected_container_pod[0], selected_container_pod[1], c_name])
retry_container_killing(kill_action, selected_container_pod[0], selected_container_pod[1], c_name)
break
else:
killed_container_list.append([selected_container_pod[0], selected_container_pod[1], c_name])
retry_container_killing(kill_action, selected_container_pod[0], selected_container_pod[1], c_name)
break
container_pod_list.remove(selected_container_pod)
killed_count += 1
logging.info("Scenario " + scenario_name + " successfully injected")
return killed_container_list
def skew_time(scenario):
skew_command = "date --set "
if scenario['action'] == "skew_date":
skewed_date = "00-01-01"
skew_command += skewed_date
elif scenario['action'] == "skew_time":
skewed_time = "01:01:01"
skew_command += skewed_time
if "node" in scenario["object_type"]:
node_names = []
if "object_name" in scenario.keys() and scenario['object_name']:
node_names = scenario['object_name']
elif "label_selector" in scenario.keys(
) and scenario['label_selector']:
node_names = kubecli.list_nodes(scenario['label_selector'])
for node in node_names:
node_debug(node, skew_command)
logging.info("Reset date/time on node " + str(node))
return "node", node_names
elif "pod" in scenario['object_type']:
pod_names = []
if "object_name" in scenario.keys() and scenario['object_name']:
for name in scenario['object_name']:
if "namespace" not in scenario.keys():
logging.error("Need to set namespace when using pod name")
sys.exit(1)
pod_names.append([name, scenario['namespace']])
elif "label_selector" in scenario.keys(
) and scenario['label_selector']:
pod_names = kubecli.get_all_pods(scenario['label_selector'])
elif "namespace" in scenario.keys() and scenario['namespace']:
pod_names = kubecli.list_pods(scenario['namespace'])
counter = 0
for pod_name in pod_names:
pod_names[counter] = [pod_name, scenario['namespace']]
counter += 1
for pod in pod_names:
if len(pod) > 1:
pod_exec(pod[0], skew_command, pod[1])
else:
pod_exec(pod, skew_command, scenario['namespace'])
logging.info("Reset date/time on pod " + str(pod[0]))
return "pod", pod_names
def skew_time(scenario):
skew_command = "date --set "
if scenario["action"] == "skew_date":
skewed_date = "00-01-01"
skew_command += skewed_date
elif scenario["action"] == "skew_time":
skewed_time = "01:01:01"
skew_command += skewed_time
if "node" in scenario["object_type"]:
node_names = []
if "object_name" in scenario.keys() and scenario["object_name"]:
node_names = scenario["object_name"]
elif "label_selector" in scenario.keys() and scenario["label_selector"]:
node_names = kubecli.list_nodes(scenario["label_selector"])
for node in node_names:
node_debug(node, skew_command)
logging.info("Reset date/time on node " + str(node))
return "node", node_names
elif "pod" in scenario["object_type"]:
container_name = scenario.get("container_name", "")
pod_names = []
if "object_name" in scenario.keys() and scenario["object_name"]:
for name in scenario["object_name"]:
if "namespace" not in scenario.keys():
logging.error("Need to set namespace when using pod name")
sys.exit(1)
pod_names.append([name, scenario["namespace"]])
elif "namespace" in scenario.keys() and scenario["namespace"]:
if "label_selector" not in scenario.keys():
logging.info(
"label_selector key not found, querying for all the pods in namespace: %s" % (scenario["namespace"])
)
pod_names = kubecli.list_pods(scenario["namespace"])
else:
logging.info(
"Querying for the pods matching the %s label_selector in namespace %s"
% (scenario["label_selector"], scenario["namespace"])
)
pod_names = kubecli.list_pods(scenario["namespace"], scenario["label_selector"])
counter = 0
for pod_name in pod_names:
pod_names[counter] = [pod_name, scenario["namespace"]]
counter += 1
elif "label_selector" in scenario.keys() and scenario["label_selector"]:
pod_names = kubecli.get_all_pods(scenario["label_selector"])
if len(pod_names) == 0:
logging.info("Cannot find pods matching the namespace/label_selector, please check")
sys.exit(1)
pod_counter = 0
for pod in pod_names:
if len(pod) > 1:
selected_container_name = get_container_name(pod[0], pod[1], container_name)
pod_exec_response = pod_exec(pod[0], skew_command, pod[1], selected_container_name)
if pod_exec_response is False:
logging.error(
"Couldn't reset time on container %s in pod %s in namespace %s"
% (selected_container_name, pod[0], pod[1])
)
sys.exit(1)
pod_names[pod_counter].append(selected_container_name)
else:
selected_container_name = get_container_name(pod, scenario["namespace"], container_name)
pod_exec_response = pod_exec(pod, skew_command, scenario["namespace"], selected_container_name)
if pod_exec_response is False:
logging.error(
"Couldn't reset time on container %s in pod %s in namespace %s"
% (selected_container_name, pod, scenario["namespace"])
)
sys.exit(1)
pod_names[pod_counter].append(selected_container_name)
logging.info("Reset date/time on pod " + str(pod[0]))
pod_counter += 1
return "pod", pod_names
def get_job_pods(api_response):
controllerUid = api_response.metadata.labels["controller-uid"]
pod_label_selector = "controller-uid=" + controllerUid
pods_list = kubecli.list_pods(label_selector=pod_label_selector,
namespace="default")
return pods_list[0]