def test_check_min_nodes_exist(client, has_conf): has_conf.return_value = False v1 = MagicMock() node1 = create_node_object("old_node") taint = k8sClient.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) node1.spec.taints = [taint] node1.metadata.creation_timestamp = datetime.datetime.now( ) - datetime.timedelta(seconds=1000) node2 = create_node_object("new_node") taint = k8sClient.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) node2.spec.taints = [taint] node2.metadata.creation_timestamp = datetime.datetime.now( ) - datetime.timedelta(seconds=10) response = k8sClient.V1NodeList(items=[node1, node2]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) resp = check_min_nodes_exist(k8s_label_selector="some_selector", min_limit=2) assert True == resp resp = check_min_nodes_exist(k8s_label_selector="some_selector", min_limit=5) assert False == resp
def test_all_pods_in_all_ns_are_ok_failure(nodes_client, client, has_conf): node_name = 'node1' has_conf.return_value = False v1 = MagicMock() pod1 = create_pod_object("fakepod1", node_name=node_name) pod2 = create_pod_object("fakepod2", state='terminated', node_name=node_name) pod2.status.container_statuses[0].running = None pod2.spec.node_name = node_name response = k8sClient.V1PodList(items=[pod1, pod2]) v1.list_pod_for_all_namespaces.return_value = response node = MagicMock() node.metadata.name = "node1" v1.list_node_with_http_info.return_value = k8sClient.V1NodeList( items=[node]) nodes_client.CoreV1Api.return_value = v1 nodes_client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) client.CoreV1Api.return_value = v1 resp = all_pods_in_all_ns_are_ok( configuration={"ns-ignore-list": ["db-catalog"]}) v1.list_pod_for_all_namespaces.assert_called_with(watch=False) assert resp is False
def test_all_pods_in_all_ns_are_ok_completed(node_client, client, has_conf): node_name = 'node1' has_conf.return_value = False v1 = MagicMock() pod1 = create_pod_object("fakepod1", node_name=node_name) pod2 = create_pod_object("fakepod2", state="terminated", node_name=node_name) pod2.status.container_statuses[0].state.terminated.reason = 'Completed' node = MagicMock() node.metadata.name = node_name v1.list_node_with_http_info.return_value = k8sClient.V1NodeList( items=[node]) response = k8sClient.V1PodList(items=[pod1, pod2]) v1.list_pod_for_all_namespaces.return_value = response client.CoreV1Api.return_value = v1 node_client.CoreV1Api.return_value = v1 node_client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) resp = all_pods_in_all_ns_are_ok(configuration={"ns-ignore-list": []}) v1.list_pod_for_all_namespaces.assert_called_with(watch=False) assert resp is True
def test_attach_sq_to_instance_by_tag(gks, client, has_conf, boto_client): has_conf.return_value = False gks.return_value = { 'url': 'fake_url.com', 'token': 'fake_token_towhatever', 'SLACK_CHANNEL': 'chaos_fanout', 'SLACK_TOKEN': 'sometoken' } v1 = MagicMock() taint1 = k8sClient.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) taint2 = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", value="spot", time_added=None) ignore_list = [taint1, taint2] node1 = create_node_object("node1") node2 = create_node_object("tainted_node_ignore") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="spot") node2.spec.taints = [taint] response = k8sClient.V1NodeList(items=[node1, node2]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) client = MagicMock() boto_client.client.return_value = client boto_client.resource.return_value = client network_interface = MagicMock() instance = MagicMock() instance.security_groups = [{"GroupId": "some_testsgid"}] instance.network_interfaces = [network_interface] client.instances.filter.return_value = [instance] client.describe_security_groups.return_value = { 'SecurityGroups': [{ 'GroupId': "i_testsgid", }] } config = create_config_with_taint_ignore() retval = attach_sq_to_instance_by_tag(tag_name="under_chaostest", sg_name="chaos_test_sg", configuration=config) assert retval is not None network_interface.modify_attribute.assert_called_with( Groups=['i_testsgid'])
def get_active_nodes(label_selector: str = None, taints_ignore_list=None, secrets: Secrets = None): """ List all nodes, that are not tainted by known taints. You may filter nodes by specifying a label selector. """ if taints_ignore_list is None: taints_ignore_list = [] api = create_k8s_api_client(secrets) v1 = client.CoreV1Api(api) if label_selector: ret = v1.list_node_with_http_info(label_selector=label_selector, _preload_content=True, _return_http_data_only=True) else: ret = v1.list_node_with_http_info(_preload_content=True, _return_http_data_only=True) node_list = ret.items retval = client.V1NodeList(items=[]) for node in node_list: node_ignored = False if node.spec.taints is not None: node_ignored = node_should_be_ignored_by_taints( node.spec.taints, taints_ignore_list) if not node_ignored: retval.items.append(node) return retval, v1
def test_add_label_to_node(cl, client, has_conf): fake_node_name = "fake_node.com" has_conf.return_value = False v1 = MagicMock() condition = k8sClient.V1NodeCondition(type="Ready", status="True") status = k8sClient.V1NodeStatus(conditions=[condition]) spec = k8sClient.V1NodeSpec(unschedulable=False) metadata = k8sClient.V1ObjectMeta(name=fake_node_name, labels={"label1": "True"}) node = k8sClient.V1Node(status=status, spec=spec, metadata=metadata) response = k8sClient.V1NodeList(items=[node]) v1.list_node_with_http_info.return_value = response v1.patch_node.return_value = node client.CoreV1Api.return_value = v1 label_selector = 'label_default=true' add_label_to_node(label_selector=label_selector, label_name="label1", label_value="value1") v1.list_node_with_http_info.assert_called_with( label_selector=label_selector, _preload_content=True, _return_http_data_only=True) v1.patch_node.assert_called_with( fake_node_name, {'metadata': { 'labels': { 'label1': "value1" } }})
def test_get_non_tainted_nodes(client, has_conf): has_conf.return_value = False v1 = MagicMock() node1 = create_node_object("tainted_node") taint = k8sClient.V1Taint(effect="NoSchedule", key="faketaint", time_added=None, value=None) node1.spec.taints = [taint] node2 = create_node_object("not_tainted_node") response = k8sClient.V1NodeList(items=[node1, node2]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) resp, v1 = get_active_nodes() assert len(resp.items) == 2
def test_get_tainted_nodes(cl, client, has_conf): has_conf.return_value = False def replace_k8s_taint(effect, key, time_added, value): return k8sClient.V1Taint(effect=effect, key=key, time_added=time_added, value=value) v1 = MagicMock() node1 = create_node_object("tainted_node") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="special") node1.spec.taints = [taint] node2 = create_node_object("tainted_node_ignore") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="spot") node2.spec.taints = [taint] node3 = create_node_object("not_tainted") response = k8sClient.V1NodeList(items=[node1, node2, node3]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) client.V1Taint.return_value = replace_k8s_taint resp = get_tainted_nodes(key="dedicated", value="special", effect="NoSchedule") assert 1 == len(resp) assert resp[0].metadata.name == node1.metadata.name
def test_get_non_tainted_nodes_filtered(cl, client, has_conf): has_conf.return_value = False v1 = MagicMock() taint1 = k8sClient.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) taint2 = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", value="spot", time_added=None) ignore_list = [taint1, taint2] node1 = create_node_object("tainted_node") taint = k8sClient.V1Taint(effect="NoSchedule", key="faketaint", time_added=None, value=None) node1.spec.taints = [taint] node2 = create_node_object("tainted_node_ignore") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="spot") node2.spec.taints = [taint] node3 = create_node_object("not_tainted") response = k8sClient.V1NodeList(items=[node1, node2, node3]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) resp, v1 = get_active_nodes(taints_ignore_list=ignore_list) assert 2 == len(resp.items)
def test_remove_label_from_node(gks, cl, client, has_conf): fake_node_name = "fake_node.com" gks.return_value = { 'url': 'fake_url.com', 'token': 'fake_token_towhatever', 'SLACK_CHANNEL': 'chaos_fanout', 'SLACK_TOKEN': 'sometoken' } has_conf.return_value = False v1 = MagicMock() condition = k8sClient.V1NodeCondition(type="Ready", status="True") status = k8sClient.V1NodeStatus(conditions=[condition]) spec = k8sClient.V1NodeSpec(unschedulable=False) metadata = k8sClient.V1ObjectMeta(name=fake_node_name, labels={"label1": "True"}) node = k8sClient.V1Node(status=status, spec=spec, metadata=metadata) response = k8sClient.V1NodeList(items=[node]) v1.list_node_with_http_info.return_value = response v1.patch_node.return_value = node client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) label_selector = 'label_default=true, label1=True' remove_label_from_node(label_selector, "label1") v1.list_node_with_http_info.assert_called_with( label_selector=label_selector, _preload_content=True, _return_http_data_only=True) v1.patch_node.assert_called_with( fake_node_name, {'metadata': { 'labels': { 'label1': None } }})
def test_all_nodes_are_ok(cl, client, has_conf): has_conf.return_value = False v1 = MagicMock() node = create_node_object("fakenode") response = k8sClient.V1NodeList(items=[node]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 label_selector = 'beta.kubernetes.io/instance-type=m5.large' resp = all_nodes_are_ok(label_selector=label_selector) v1.list_node_with_http_info.assert_called_with( label_selector=label_selector, _preload_content=True, _return_http_data_only=True) assert resp is True
def test_can_select_nodes_by_label(cl, client, has_conf): has_conf.return_value = False v1 = MagicMock() condition = k8sClient.V1NodeCondition(type="Ready", status="True") status = k8sClient.V1NodeStatus(conditions=[condition]) node = k8sClient.V1Node(status=status) response = k8sClient.V1NodeList(items=[node]) v1.list_node.return_value = response client.CoreV1Api.return_value = v1 label_selector = 'beta.kubernetes.io/instance-type=m5.large' resp = all_nodes_are_ok(label_selector=label_selector) v1.list_node.assert_called_with( label_selector=label_selector, _preload_content=False) assert resp is True
def test_tag_random_node_aws(clientApi, has_conf, boto_client): has_conf.return_value = False v1 = MagicMock() node1 = create_node_object("node1") node2 = create_node_object("tainted_node_ignore") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="spot") node2.spec.taints = [taint] response = k8sClient.V1NodeList(items=[node1, node2]) v1.list_node_with_http_info.return_value = response clientApi.return_value = v1 client = MagicMock() boto_client.client.return_value = client client.describe_instances.return_value = { 'Reservations': [{ 'Instances': [{ 'InstanceId': "id_1", 'InstanceLifecycle': 'normal', 'PrivateDnsName': 'node1' }] }] } config = create_config_with_taint_ignore() retval, nodename = tag_random_node_aws(k8s_label_selector="label_selector", secrets=None, tag_name="test_tag", configuration=config) assert retval == 0 assert nodename == "node1" client.create_tags.assert_called_with(Resources=['id_1'], Tags=[{ 'Key': 'test_tag', 'Value': 'test_tag' }])
def test_taint_nodes_by_label(gks, cl, client, has_conf): fake_node_name = "fake_node.com" gks.return_value = { 'url': 'fake_url.com', 'token': 'fake_token_towhatever', 'SLACK_CHANNEL': 'chaos_fanout', 'SLACK_TOKEN': 'sometoken' } has_conf.return_value = False v1 = MagicMock() condition = k8sClient.V1NodeCondition(type="Ready", status="True") status = k8sClient.V1NodeStatus(conditions=[condition]) spec = k8sClient.V1NodeSpec(unschedulable=False) metadata = k8sClient.V1ObjectMeta(name=fake_node_name, labels={"label1": "True"}) node = k8sClient.V1Node(status=status, spec=spec, metadata=metadata) response = k8sClient.V1NodeList(items=[node]) v1.list_node_with_http_info.return_value = response v1.patch_node.return_value = node client.CoreV1Api.return_value = v1 client.V1Taint.return_value = k8sClient.V1Taint(key="", value="", effect="") label_selector = 'label_default=true, label1=True' taint_nodes_by_label(label_selector=label_selector, key="key1", value="Apps", effect="NoExec") assert v1.patch_node.call_count == 1 args = v1.patch_node.call_args[0] assert args[0] == fake_node_name assert args[1]['spec']['taints'][0].key == "key1" assert args[1]['spec']['taints'][0].effect == "NoExec" assert args[1]['spec']['taints'][0].value == "Apps"
def test_iptables_block_port_no_taint_only(gks, fabric, client, has_conf, boto_client): fabric_api = MagicMock() fabric.return_value = fabric_api gks.return_value = { 'url': 'fake_url.com', 'token': 'fake_token_towhatever', 'SLACK_CHANNEL': 'chaos_fanout', 'SLACK_TOKEN': 'sometoken' } os.environ["SSH_KEY"] = "keytext" os.environ["SSH_USER"] = "******" has_conf.return_value = False v1 = MagicMock() taint1 = k8sClient.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) taint2 = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", value="spot", time_added=None) ignore_list = [taint1, taint2] node1 = create_node_object("node1") node2 = create_node_object("tainted_node_ignore") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="spot") node2.spec.taints = [taint] response = k8sClient.V1NodeList(items=[node1, node2]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) client = MagicMock() boto_client.client.return_value = client boto_client.resource.return_value = client instance = MagicMock() instance.pivate_ip_address = "test_ip" instance.security_groups = [{"GroupId": "some_testsgid"}] client.instances.filter.return_value = [instance] client.describe_security_groups.return_value = { 'SecurityGroups': [{ 'GroupId': "i_testsgid", }] } config = create_config_with_taint_ignore() retval = iptables_block_port(tag_name="under_chaostest", port=53, protocols=["tcp"], configuration=config) assert retval is not None text = "iptables -I PREROUTING -t nat -p {} --dport {} -j DNAT --to-destination 0.0.0.0:1000".format( "tcp", 53) fabric.sudo.assert_called_with(text)