def test_check_min_nodes_exist(client, has_conf):
has_conf.return_value = False
v1 = MagicMock()
node1 = create_node_object("old_node")
taint = k8sClient.V1Taint(effect="NoSchedule",
key="node-role.kubernetes.io/master",
value=None,
time_added=None)
node1.spec.taints = [taint]
node1.metadata.creation_timestamp = datetime.datetime.now(
) - datetime.timedelta(seconds=1000)
node2 = create_node_object("new_node")
taint = k8sClient.V1Taint(effect="NoSchedule",
key="node-role.kubernetes.io/master",
value=None,
time_added=None)
node2.spec.taints = [taint]
node2.metadata.creation_timestamp = datetime.datetime.now(
) - datetime.timedelta(seconds=10)
response = k8sClient.V1NodeList(items=[node1, node2])
v1.list_node_with_http_info.return_value = response
client.CoreV1Api.return_value = v1
client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])
resp = check_min_nodes_exist(k8s_label_selector="some_selector",
min_limit=2)
assert True == resp
resp = check_min_nodes_exist(k8s_label_selector="some_selector",
min_limit=5)
assert False == resp
def test_all_pods_in_all_ns_are_ok_failure(nodes_client, client, has_conf):
node_name = 'node1'
has_conf.return_value = False
v1 = MagicMock()
pod1 = create_pod_object("fakepod1", node_name=node_name)
pod2 = create_pod_object("fakepod2",
state='terminated',
node_name=node_name)
pod2.status.container_statuses[0].running = None
pod2.spec.node_name = node_name
response = k8sClient.V1PodList(items=[pod1, pod2])
v1.list_pod_for_all_namespaces.return_value = response
node = MagicMock()
node.metadata.name = "node1"
v1.list_node_with_http_info.return_value = k8sClient.V1NodeList(
items=[node])
nodes_client.CoreV1Api.return_value = v1
nodes_client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])
client.CoreV1Api.return_value = v1
resp = all_pods_in_all_ns_are_ok(
configuration={"ns-ignore-list": ["db-catalog"]})
v1.list_pod_for_all_namespaces.assert_called_with(watch=False)
assert resp is False
def test_all_pods_in_all_ns_are_ok_completed(node_client, client, has_conf):
node_name = 'node1'
has_conf.return_value = False
v1 = MagicMock()
pod1 = create_pod_object("fakepod1", node_name=node_name)
pod2 = create_pod_object("fakepod2",
state="terminated",
node_name=node_name)
pod2.status.container_statuses[0].state.terminated.reason = 'Completed'
node = MagicMock()
node.metadata.name = node_name
v1.list_node_with_http_info.return_value = k8sClient.V1NodeList(
items=[node])
response = k8sClient.V1PodList(items=[pod1, pod2])
v1.list_pod_for_all_namespaces.return_value = response
client.CoreV1Api.return_value = v1
node_client.CoreV1Api.return_value = v1
node_client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])
resp = all_pods_in_all_ns_are_ok(configuration={"ns-ignore-list": []})
v1.list_pod_for_all_namespaces.assert_called_with(watch=False)
assert resp is True
def test_attach_sq_to_instance_by_tag(gks, client, has_conf, boto_client):
has_conf.return_value = False
gks.return_value = {
'url': 'fake_url.com',
'token': 'fake_token_towhatever',
'SLACK_CHANNEL': 'chaos_fanout',
'SLACK_TOKEN': 'sometoken'
}
v1 = MagicMock()
taint1 = k8sClient.V1Taint(effect="NoSchedule",
key="node-role.kubernetes.io/master",
value=None,
time_added=None)
taint2 = k8sClient.V1Taint(effect="NoSchedule",
key="dedicated",
value="spot",
time_added=None)
ignore_list = [taint1, taint2]
node1 = create_node_object("node1")
node2 = create_node_object("tainted_node_ignore")
taint = k8sClient.V1Taint(effect="NoSchedule",
key="dedicated",
time_added=None,
value="spot")
node2.spec.taints = [taint]
response = k8sClient.V1NodeList(items=[node1, node2])
v1.list_node_with_http_info.return_value = response
client.CoreV1Api.return_value = v1
client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])
client = MagicMock()
boto_client.client.return_value = client
boto_client.resource.return_value = client
network_interface = MagicMock()
instance = MagicMock()
instance.security_groups = [{"GroupId": "some_testsgid"}]
instance.network_interfaces = [network_interface]
client.instances.filter.return_value = [instance]
client.describe_security_groups.return_value = {
'SecurityGroups': [{
'GroupId': "i_testsgid",
}]
}
config = create_config_with_taint_ignore()
retval = attach_sq_to_instance_by_tag(tag_name="under_chaostest",
sg_name="chaos_test_sg",
configuration=config)
assert retval is not None
network_interface.modify_attribute.assert_called_with(
Groups=['i_testsgid'])
def get_active_nodes(label_selector: str = None,
taints_ignore_list=None,
secrets: Secrets = None):
"""
List all nodes, that are not tainted by known taints. You may filter nodes
by specifying a label selector.
"""
if taints_ignore_list is None:
taints_ignore_list = []
api = create_k8s_api_client(secrets)
v1 = client.CoreV1Api(api)
if label_selector:
ret = v1.list_node_with_http_info(label_selector=label_selector,
_preload_content=True,
_return_http_data_only=True)
else:
ret = v1.list_node_with_http_info(_preload_content=True,
_return_http_data_only=True)
node_list = ret.items
retval = client.V1NodeList(items=[])
for node in node_list:
node_ignored = False
if node.spec.taints is not None:
node_ignored = node_should_be_ignored_by_taints(
node.spec.taints, taints_ignore_list)
if not node_ignored:
retval.items.append(node)
return retval, v1
def test_add_label_to_node(cl, client, has_conf):
fake_node_name = "fake_node.com"
has_conf.return_value = False
v1 = MagicMock()
condition = k8sClient.V1NodeCondition(type="Ready", status="True")
status = k8sClient.V1NodeStatus(conditions=[condition])
spec = k8sClient.V1NodeSpec(unschedulable=False)
metadata = k8sClient.V1ObjectMeta(name=fake_node_name,
labels={"label1": "True"})
node = k8sClient.V1Node(status=status, spec=spec, metadata=metadata)
response = k8sClient.V1NodeList(items=[node])
v1.list_node_with_http_info.return_value = response
v1.patch_node.return_value = node
client.CoreV1Api.return_value = v1
label_selector = 'label_default=true'
add_label_to_node(label_selector=label_selector,
label_name="label1",
label_value="value1")
v1.list_node_with_http_info.assert_called_with(
label_selector=label_selector,
_preload_content=True,
_return_http_data_only=True)
v1.patch_node.assert_called_with(
fake_node_name, {'metadata': {
'labels': {
'label1': "value1"
}
}})
def test_get_non_tainted_nodes(client, has_conf):
has_conf.return_value = False
v1 = MagicMock()
node1 = create_node_object("tainted_node")
taint = k8sClient.V1Taint(effect="NoSchedule",
key="faketaint",
time_added=None,
value=None)
node1.spec.taints = [taint]
node2 = create_node_object("not_tainted_node")
response = k8sClient.V1NodeList(items=[node1, node2])
v1.list_node_with_http_info.return_value = response
client.CoreV1Api.return_value = v1
client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])
resp, v1 = get_active_nodes()
assert len(resp.items) == 2
def test_get_tainted_nodes(cl, client, has_conf):
has_conf.return_value = False
def replace_k8s_taint(effect, key, time_added, value):
return k8sClient.V1Taint(effect=effect,
key=key,
time_added=time_added,
value=value)
v1 = MagicMock()
node1 = create_node_object("tainted_node")
taint = k8sClient.V1Taint(effect="NoSchedule",
key="dedicated",
time_added=None,
value="special")
node1.spec.taints = [taint]
node2 = create_node_object("tainted_node_ignore")
taint = k8sClient.V1Taint(effect="NoSchedule",
key="dedicated",
time_added=None,
value="spot")
node2.spec.taints = [taint]
node3 = create_node_object("not_tainted")
response = k8sClient.V1NodeList(items=[node1, node2, node3])
v1.list_node_with_http_info.return_value = response
client.CoreV1Api.return_value = v1
client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])
client.V1Taint.return_value = replace_k8s_taint
resp = get_tainted_nodes(key="dedicated",
value="special",
effect="NoSchedule")
assert 1 == len(resp)
assert resp[0].metadata.name == node1.metadata.name
def test_get_non_tainted_nodes_filtered(cl, client, has_conf):
has_conf.return_value = False
v1 = MagicMock()
taint1 = k8sClient.V1Taint(effect="NoSchedule",
key="node-role.kubernetes.io/master",
value=None,
time_added=None)
taint2 = k8sClient.V1Taint(effect="NoSchedule",
key="dedicated",
value="spot",
time_added=None)
ignore_list = [taint1, taint2]
node1 = create_node_object("tainted_node")
taint = k8sClient.V1Taint(effect="NoSchedule",
key="faketaint",
time_added=None,
value=None)
node1.spec.taints = [taint]
node2 = create_node_object("tainted_node_ignore")
taint = k8sClient.V1Taint(effect="NoSchedule",
key="dedicated",
time_added=None,
value="spot")
node2.spec.taints = [taint]
node3 = create_node_object("not_tainted")
response = k8sClient.V1NodeList(items=[node1, node2, node3])
v1.list_node_with_http_info.return_value = response
client.CoreV1Api.return_value = v1
client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])
resp, v1 = get_active_nodes(taints_ignore_list=ignore_list)
assert 2 == len(resp.items)
def test_remove_label_from_node(gks, cl, client, has_conf):
fake_node_name = "fake_node.com"
gks.return_value = {
'url': 'fake_url.com',
'token': 'fake_token_towhatever',
'SLACK_CHANNEL': 'chaos_fanout',
'SLACK_TOKEN': 'sometoken'
}
has_conf.return_value = False
v1 = MagicMock()
condition = k8sClient.V1NodeCondition(type="Ready", status="True")
status = k8sClient.V1NodeStatus(conditions=[condition])
spec = k8sClient.V1NodeSpec(unschedulable=False)
metadata = k8sClient.V1ObjectMeta(name=fake_node_name,
labels={"label1": "True"})
node = k8sClient.V1Node(status=status, spec=spec, metadata=metadata)
response = k8sClient.V1NodeList(items=[node])
v1.list_node_with_http_info.return_value = response
v1.patch_node.return_value = node
client.CoreV1Api.return_value = v1
client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])
label_selector = 'label_default=true, label1=True'
remove_label_from_node(label_selector, "label1")
v1.list_node_with_http_info.assert_called_with(
label_selector=label_selector,
_preload_content=True,
_return_http_data_only=True)
v1.patch_node.assert_called_with(
fake_node_name, {'metadata': {
'labels': {
'label1': None
}
}})
def test_all_nodes_are_ok(cl, client, has_conf):
has_conf.return_value = False
v1 = MagicMock()
node = create_node_object("fakenode")
response = k8sClient.V1NodeList(items=[node])
v1.list_node_with_http_info.return_value = response
client.CoreV1Api.return_value = v1
label_selector = 'beta.kubernetes.io/instance-type=m5.large'
resp = all_nodes_are_ok(label_selector=label_selector)
v1.list_node_with_http_info.assert_called_with(
label_selector=label_selector,
_preload_content=True,
_return_http_data_only=True)
assert resp is True
def test_can_select_nodes_by_label(cl, client, has_conf):
has_conf.return_value = False
v1 = MagicMock()
condition = k8sClient.V1NodeCondition(type="Ready", status="True")
status = k8sClient.V1NodeStatus(conditions=[condition])
node = k8sClient.V1Node(status=status)
response = k8sClient.V1NodeList(items=[node])
v1.list_node.return_value = response
client.CoreV1Api.return_value = v1
label_selector = 'beta.kubernetes.io/instance-type=m5.large'
resp = all_nodes_are_ok(label_selector=label_selector)
v1.list_node.assert_called_with(
label_selector=label_selector, _preload_content=False)
assert resp is True
def test_tag_random_node_aws(clientApi, has_conf, boto_client):
has_conf.return_value = False
v1 = MagicMock()
node1 = create_node_object("node1")
node2 = create_node_object("tainted_node_ignore")
taint = k8sClient.V1Taint(effect="NoSchedule",
key="dedicated",
time_added=None,
value="spot")
node2.spec.taints = [taint]
response = k8sClient.V1NodeList(items=[node1, node2])
v1.list_node_with_http_info.return_value = response
clientApi.return_value = v1
client = MagicMock()
boto_client.client.return_value = client
client.describe_instances.return_value = {
'Reservations': [{
'Instances': [{
'InstanceId': "id_1",
'InstanceLifecycle': 'normal',
'PrivateDnsName': 'node1'
}]
}]
}
config = create_config_with_taint_ignore()
retval, nodename = tag_random_node_aws(k8s_label_selector="label_selector",
secrets=None,
tag_name="test_tag",
configuration=config)
assert retval == 0
assert nodename == "node1"
client.create_tags.assert_called_with(Resources=['id_1'],
Tags=[{
'Key': 'test_tag',
'Value': 'test_tag'
}])
def test_taint_nodes_by_label(gks, cl, client, has_conf):
fake_node_name = "fake_node.com"
gks.return_value = {
'url': 'fake_url.com',
'token': 'fake_token_towhatever',
'SLACK_CHANNEL': 'chaos_fanout',
'SLACK_TOKEN': 'sometoken'
}
has_conf.return_value = False
v1 = MagicMock()
condition = k8sClient.V1NodeCondition(type="Ready", status="True")
status = k8sClient.V1NodeStatus(conditions=[condition])
spec = k8sClient.V1NodeSpec(unschedulable=False)
metadata = k8sClient.V1ObjectMeta(name=fake_node_name,
labels={"label1": "True"})
node = k8sClient.V1Node(status=status, spec=spec, metadata=metadata)
response = k8sClient.V1NodeList(items=[node])
v1.list_node_with_http_info.return_value = response
v1.patch_node.return_value = node
client.CoreV1Api.return_value = v1
client.V1Taint.return_value = k8sClient.V1Taint(key="",
value="",
effect="")
label_selector = 'label_default=true, label1=True'
taint_nodes_by_label(label_selector=label_selector,
key="key1",
value="Apps",
effect="NoExec")
assert v1.patch_node.call_count == 1
args = v1.patch_node.call_args[0]
assert args[0] == fake_node_name
assert args[1]['spec']['taints'][0].key == "key1"
assert args[1]['spec']['taints'][0].effect == "NoExec"
assert args[1]['spec']['taints'][0].value == "Apps"
def test_iptables_block_port_no_taint_only(gks, fabric, client, has_conf,
boto_client):
fabric_api = MagicMock()
fabric.return_value = fabric_api
gks.return_value = {
'url': 'fake_url.com',
'token': 'fake_token_towhatever',
'SLACK_CHANNEL': 'chaos_fanout',
'SLACK_TOKEN': 'sometoken'
}
os.environ["SSH_KEY"] = "keytext"
os.environ["SSH_USER"] = "******"
has_conf.return_value = False
v1 = MagicMock()
taint1 = k8sClient.V1Taint(effect="NoSchedule",
key="node-role.kubernetes.io/master",
value=None,
time_added=None)
taint2 = k8sClient.V1Taint(effect="NoSchedule",
key="dedicated",
value="spot",
time_added=None)
ignore_list = [taint1, taint2]
node1 = create_node_object("node1")
node2 = create_node_object("tainted_node_ignore")
taint = k8sClient.V1Taint(effect="NoSchedule",
key="dedicated",
time_added=None,
value="spot")
node2.spec.taints = [taint]
response = k8sClient.V1NodeList(items=[node1, node2])
v1.list_node_with_http_info.return_value = response
client.CoreV1Api.return_value = v1
client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])
client = MagicMock()
boto_client.client.return_value = client
boto_client.resource.return_value = client
instance = MagicMock()
instance.pivate_ip_address = "test_ip"
instance.security_groups = [{"GroupId": "some_testsgid"}]
client.instances.filter.return_value = [instance]
client.describe_security_groups.return_value = {
'SecurityGroups': [{
'GroupId': "i_testsgid",
}]
}
config = create_config_with_taint_ignore()
retval = iptables_block_port(tag_name="under_chaostest",
port=53,
protocols=["tcp"],
configuration=config)
assert retval is not None
text = "iptables -I PREROUTING -t nat -p {} --dport {} -j DNAT --to-destination 0.0.0.0:1000".format(
"tcp", 53)
fabric.sudo.assert_called_with(text)
