def test_check_min_nodes_exist(client, has_conf): has_conf.return_value = False v1 = MagicMock() node1 = create_node_object("old_node") taint = k8sClient.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) node1.spec.taints = [taint] node1.metadata.creation_timestamp = datetime.datetime.now( ) - datetime.timedelta(seconds=1000) node2 = create_node_object("new_node") taint = k8sClient.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) node2.spec.taints = [taint] node2.metadata.creation_timestamp = datetime.datetime.now( ) - datetime.timedelta(seconds=10) response = k8sClient.V1NodeList(items=[node1, node2]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) resp = check_min_nodes_exist(k8s_label_selector="some_selector", min_limit=2) assert True == resp resp = check_min_nodes_exist(k8s_label_selector="some_selector", min_limit=5) assert False == resp
def test_attach_sq_to_instance_by_tag(gks, client, has_conf, boto_client): has_conf.return_value = False gks.return_value = { 'url': 'fake_url.com', 'token': 'fake_token_towhatever', 'SLACK_CHANNEL': 'chaos_fanout', 'SLACK_TOKEN': 'sometoken' } v1 = MagicMock() taint1 = k8sClient.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) taint2 = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", value="spot", time_added=None) ignore_list = [taint1, taint2] node1 = create_node_object("node1") node2 = create_node_object("tainted_node_ignore") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="spot") node2.spec.taints = [taint] response = k8sClient.V1NodeList(items=[node1, node2]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) client = MagicMock() boto_client.client.return_value = client boto_client.resource.return_value = client network_interface = MagicMock() instance = MagicMock() instance.security_groups = [{"GroupId": "some_testsgid"}] instance.network_interfaces = [network_interface] client.instances.filter.return_value = [instance] client.describe_security_groups.return_value = { 'SecurityGroups': [{ 'GroupId': "i_testsgid", }] } config = create_config_with_taint_ignore() retval = attach_sq_to_instance_by_tag(tag_name="under_chaostest", sg_name="chaos_test_sg", configuration=config) assert retval is not None network_interface.modify_attribute.assert_called_with( Groups=['i_testsgid'])
def remove_taint_from_node( label_selector: str = None, key: str = None, value: str = None, effect: str = None, secrets: Secrets = None, ) -> bool: """ remove taint from nodes by label.As rollback """ # To avoid implementation of non default constructor in tests taint_to_remove = client.V1Taint(key="", effect="", value="") taint_to_remove.key = key taint_to_remove.effect = effect taint_to_remove.value = value items, k8s_pai_v1 = get_node_list(label_selector, secrets) for node in items: try: logger.warning("Remove taint from node :" + node.metadata.name) if node.spec is not None and node.spec is not None and node.spec.taints is not None: existing_taints = node.spec.taints patch = generate_patch_for_taint_deletion( existing_taints, taint_to_remove) k8s_pai_v1.patch_node(node.metadata.name, patch) except ApiException as x: raise FailedActivity("Un tainting node failed: {}".format(x.body)) return True
def taint_nodes_by_label(label_selector: str = None, key: str = None, value: str = None, effect: str = None, secrets: Secrets = None) -> bool: """ taint nodes by label. """ # To avoid implementation of non default constructor in tests new_taint = client.V1Taint(key="", effect="", value="") new_taint.key = key new_taint.effect = effect new_taint.value = value items, k8s_api_v1 = get_node_list(label_selector, secrets) for node in items: try: logger.warning("Taint node :" + node.metadata.name) existing_taints = [] if node.spec is not None and node.spec is not None and node.spec.taints is not None: existing_taints = node.spec.taints body_patch = generate_patch_for_taint(existing_taints, new_taint) k8s_api_v1.patch_node(node.metadata.name, body_patch) except ApiException as x: raise FailedActivity("tainting node failed: {}".format(x.body)) return True
def taint_node(node_name): """ Taint a kubernetes node to avoid new pods being scheduled on it """ try: config.load_incluster_config() except config.ConfigException: try: config.load_kube_config() except config.ConfigException: raise Exception("Could not configure kubernetes python client") configuration = client.Configuration() # create an instance of the API class k8s_api = client.CoreV1Api(client.ApiClient(configuration)) logger.info("Adding taint to k8s node {}...".format(node_name)) try: taint = client.V1Taint(effect='NoSchedule', key='eks-rolling-update') api_call_body = client.V1Node(spec=client.V1NodeSpec(taints=[taint])) if not app_config['DRY_RUN']: k8s_api.patch_node(node_name, api_call_body) else: k8s_api.patch_node(node_name, api_call_body, dry_run=True) logger.info("Added taint to the node") except ApiException as e: logger.info("Exception when calling CoreV1Api->patch_node: {}".format(e))
def get_tainted_nodes(key: str = None, value: str = None, effect: str = None, secrets: Secrets = None): """ Get nodes that are tainted with specific taint. :param key: Taint key :param value: Taint value :param effect: Taint effect :param secrets: chaostoolkit will inject secrets :return: array of nodes with specific taint """ all_nodes, k8s_api_v1 = get_active_nodes(label_selector="", taints_ignore_list=None, secrets=secrets) taint_to_find = client.V1Taint(effect=effect, key=key, time_added=None, value=value) retval = [] for node in all_nodes.items: if node.spec is not None and node.spec.taints is not None: for taint in node.spec.taints: if is_equal_V1Taint(taint, taint_to_find): retval.append(node) return retval
def test_generate_patch_for_taint_already_exists(): taint1 = k8sClient.V1Taint(effect="NoSchedule", key="com.wixpress.somekey", time_added=None, value="ssr") taint2 = k8sClient.V1Taint(effect="noExecute", key="com.wixpress.somekey", time_added=None, value="dbmng") taint_new = k8sClient.V1Taint(effect="noExecute", key="com.wixpress.somekey", time_added=None, value="dbmng") existing_taints = [taint1, taint2] patch = generate_patch_for_taint(existing_taints, taint_new) assert len(patch['spec']['taints']) is 2
def test_get_non_tainted_nodes_filtered(cl, client, has_conf): has_conf.return_value = False v1 = MagicMock() taint1 = k8sClient.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) taint2 = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", value="spot", time_added=None) ignore_list = [taint1, taint2] node1 = create_node_object("tainted_node") taint = k8sClient.V1Taint(effect="NoSchedule", key="faketaint", time_added=None, value=None) node1.spec.taints = [taint] node2 = create_node_object("tainted_node_ignore") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="spot") node2.spec.taints = [taint] node3 = create_node_object("not_tainted") response = k8sClient.V1NodeList(items=[node1, node2, node3]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) resp, v1 = get_active_nodes(taints_ignore_list=ignore_list) assert 2 == len(resp.items)
def test_get_tainted_nodes(cl, client, has_conf): has_conf.return_value = False def replace_k8s_taint(effect, key, time_added, value): return k8sClient.V1Taint(effect=effect, key=key, time_added=time_added, value=value) v1 = MagicMock() node1 = create_node_object("tainted_node") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="special") node1.spec.taints = [taint] node2 = create_node_object("tainted_node_ignore") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="spot") node2.spec.taints = [taint] node3 = create_node_object("not_tainted") response = k8sClient.V1NodeList(items=[node1, node2, node3]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) client.V1Taint.return_value = replace_k8s_taint resp = get_tainted_nodes(key="dedicated", value="special", effect="NoSchedule") assert 1 == len(resp) assert resp[0].metadata.name == node1.metadata.name
def load_taint_list_from_dict(list_of_objects): """ Convert list of dictionaries loaded from configuration to V1Taint objects :param list_of_objects: array of objects. Usually loaded from configuration :return: array of V1Taint objects """ retval = [] for obj in list_of_objects: taint = client.V1Taint(effect=obj.get("effect"), value=obj.get("value"), key=obj.get("key"), time_added=obj.get("time_added")) retval.append(taint) return retval
def taint_node(node, key, value, effect='NoSchedule'): node_name = node.metadata.name taints = node.spec.taints or [] assert not any(taint.key == key for taint in taints) taint = client.V1Taint( key=key, effect=effect, time_added=datetime.now(tz=tzutc()), ) taints.append(taint) patch = {'spec': {'taints': taints}} v1 = client.CoreV1Api() rtn = v1.patch_node(node_name, patch) return rtn
def test_tag_random_node_aws(clientApi, has_conf, boto_client): has_conf.return_value = False v1 = MagicMock() node1 = create_node_object("node1") node2 = create_node_object("tainted_node_ignore") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="spot") node2.spec.taints = [taint] response = k8sClient.V1NodeList(items=[node1, node2]) v1.list_node_with_http_info.return_value = response clientApi.return_value = v1 client = MagicMock() boto_client.client.return_value = client client.describe_instances.return_value = { 'Reservations': [{ 'Instances': [{ 'InstanceId': "id_1", 'InstanceLifecycle': 'normal', 'PrivateDnsName': 'node1' }] }] } config = create_config_with_taint_ignore() retval, nodename = tag_random_node_aws(k8s_label_selector="label_selector", secrets=None, tag_name="test_tag", configuration=config) assert retval == 0 assert nodename == "node1" client.create_tags.assert_called_with(Resources=['id_1'], Tags=[{ 'Key': 'test_tag', 'Value': 'test_tag' }])
def test_get_non_tainted_nodes(client, has_conf): has_conf.return_value = False v1 = MagicMock() node1 = create_node_object("tainted_node") taint = k8sClient.V1Taint(effect="NoSchedule", key="faketaint", time_added=None, value=None) node1.spec.taints = [taint] node2 = create_node_object("not_tainted_node") response = k8sClient.V1NodeList(items=[node1, node2]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) resp, v1 = get_active_nodes() assert len(resp.items) == 2
def taint_node(node_name): """ Taint a kubernetes node to avoid new pods being scheduled on it """ ensure_config_loaded() k8s_api = client.CoreV1Api() logger.info("Adding taint to k8s node {}...".format(node_name)) try: taint = client.V1Taint(effect='NoSchedule', key='eks-rolling-update') api_call_body = client.V1Node(spec=client.V1NodeSpec(taints=[taint])) if not app_config['DRY_RUN']: k8s_api.patch_node(node_name, api_call_body) else: k8s_api.patch_node(node_name, api_call_body, dry_run=True) logger.info("Added taint to the node") except ApiException as e: logger.info("Exception when calling CoreV1Api->patch_node: {}".format(e))
def test_taint_nodes_by_label(gks, cl, client, has_conf): fake_node_name = "fake_node.com" gks.return_value = { 'url': 'fake_url.com', 'token': 'fake_token_towhatever', 'SLACK_CHANNEL': 'chaos_fanout', 'SLACK_TOKEN': 'sometoken' } has_conf.return_value = False v1 = MagicMock() condition = k8sClient.V1NodeCondition(type="Ready", status="True") status = k8sClient.V1NodeStatus(conditions=[condition]) spec = k8sClient.V1NodeSpec(unschedulable=False) metadata = k8sClient.V1ObjectMeta(name=fake_node_name, labels={"label1": "True"}) node = k8sClient.V1Node(status=status, spec=spec, metadata=metadata) response = k8sClient.V1NodeList(items=[node]) v1.list_node_with_http_info.return_value = response v1.patch_node.return_value = node client.CoreV1Api.return_value = v1 client.V1Taint.return_value = k8sClient.V1Taint(key="", value="", effect="") label_selector = 'label_default=true, label1=True' taint_nodes_by_label(label_selector=label_selector, key="key1", value="Apps", effect="NoExec") assert v1.patch_node.call_count == 1 args = v1.patch_node.call_args[0] assert args[0] == fake_node_name assert args[1]['spec']['taints'][0].key == "key1" assert args[1]['spec']['taints'][0].effect == "NoExec" assert args[1]['spec']['taints'][0].value == "Apps"
def test_is_equal_v1taint_ok(): taint1 = client.V1Taint(effect="NoSchedule", key="dedicated", value="spot") taint2 = client.V1Taint(effect="NoSchedule", key="dedicated", value="spot") assert is_equal_V1Taint(taint1, taint2) is True
def test_node_is_tainted_false(): ignore_list = load_taint_list_from_dict(json.loads(taint_ignore_list_text)["taints-ignore-list"]) taint = client.V1Taint(effect="NoSchedule", key="somekey", value=None, time_added=None) assert node_should_be_ignored_by_taints([taint], ignore_list) is False
def test_node_is_tainted_true(): ignore_list = load_taint_list_from_dict(json.loads(taint_ignore_list_text)["taints-ignore-list"]) taint = client.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) assert node_should_be_ignored_by_taints([taint], ignore_list) is True
def taint_node(self, name, key, value, effect): body = client.V1Node(spec=client.V1NodeSpec( taints=[client.V1Taint(key=key, value=value, effect=effect)])) return self._v1.patch_node(name, body)
def replace_k8s_taint(effect, key, time_added, value): return k8sClient.V1Taint(effect=effect, key=key, time_added=time_added, value=value)
def test_iptables_block_port_no_taint_only(gks, fabric, client, has_conf, boto_client): fabric_api = MagicMock() fabric.return_value = fabric_api gks.return_value = { 'url': 'fake_url.com', 'token': 'fake_token_towhatever', 'SLACK_CHANNEL': 'chaos_fanout', 'SLACK_TOKEN': 'sometoken' } os.environ["SSH_KEY"] = "keytext" os.environ["SSH_USER"] = "******" has_conf.return_value = False v1 = MagicMock() taint1 = k8sClient.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None) taint2 = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", value="spot", time_added=None) ignore_list = [taint1, taint2] node1 = create_node_object("node1") node2 = create_node_object("tainted_node_ignore") taint = k8sClient.V1Taint(effect="NoSchedule", key="dedicated", time_added=None, value="spot") node2.spec.taints = [taint] response = k8sClient.V1NodeList(items=[node1, node2]) v1.list_node_with_http_info.return_value = response client.CoreV1Api.return_value = v1 client.V1NodeList.return_value = k8sClient.V1NodeList(items=[]) client = MagicMock() boto_client.client.return_value = client boto_client.resource.return_value = client instance = MagicMock() instance.pivate_ip_address = "test_ip" instance.security_groups = [{"GroupId": "some_testsgid"}] client.instances.filter.return_value = [instance] client.describe_security_groups.return_value = { 'SecurityGroups': [{ 'GroupId': "i_testsgid", }] } config = create_config_with_taint_ignore() retval = iptables_block_port(tag_name="under_chaostest", port=53, protocols=["tcp"], configuration=config) assert retval is not None text = "iptables -I PREROUTING -t nat -p {} --dport {} -j DNAT --to-destination 0.0.0.0:1000".format( "tcp", 53) fabric.sudo.assert_called_with(text)
def test_generate_action_plan(self): mock_input = { 'all_nodes': [ client.V1Node(metadata=client.V1ObjectMeta( name='node-1', creation_timestamp=(now - datetime.timedelta(days=30.1)), annotations={} ), spec=client.V1NodeSpec(unschedulable=False)), client.V1Node(metadata=client.V1ObjectMeta( name='node-2', creation_timestamp=(now - datetime.timedelta(days=60)), annotations={} ), spec=client.V1NodeSpec(unschedulable=False)), client.V1Node(metadata=client.V1ObjectMeta( name='node-3', creation_timestamp=(now - datetime.timedelta(days=30.2)), annotations={} ), spec=client.V1NodeSpec(unschedulable=True)), client.V1Node(metadata=client.V1ObjectMeta( name='node-4', creation_timestamp=(now - datetime.timedelta(days=30.3)), annotations={ main.annotation('cordoned'): '', } ), spec=client.V1NodeSpec(unschedulable=False)), client.V1Node(metadata=client.V1ObjectMeta( name='node-4', creation_timestamp=(now - datetime.timedelta(days=30.4)), annotations={ main.annotation('cordoned'): '', } ), spec=client.V1NodeSpec( unschedulable=True, taints=[ client.V1Taint( key='node.kubernetes.io/unschedulable', effect='NoSchedule', time_added=(now - datetime.timedelta(hours=1)), ) ] )), client.V1Node(metadata=client.V1ObjectMeta( name='node-5', creation_timestamp=(now - datetime.timedelta(days=32.5)), annotations={ main.annotation('cordoned'): '', } ), spec=client.V1NodeSpec( unschedulable=True, taints=[ client.V1Taint( key='node.kubernetes.io/unschedulable', effect='NoSchedule', time_added=(now - datetime.timedelta(days=1.2)), ) ] )), client.V1Node(metadata=client.V1ObjectMeta( name='node-6', creation_timestamp=(now - datetime.timedelta(days=35)), annotations={ main.annotation('cordoned'): '', } ), spec=client.V1NodeSpec( unschedulable=True, taints=[ client.V1Taint( key='node.kubernetes.io/unschedulable', effect='NoSchedule', time_added=(now - datetime.timedelta(days=2)), ) ] )), client.V1Node(metadata=client.V1ObjectMeta( name='node-7', creation_timestamp=(now - datetime.timedelta(days=35)), annotations={ main.annotation('cordoned'): '', main.annotation('notifications-sent'): str(int((datetime.datetime.utcnow() - datetime.timedelta(days=2.5)).timestamp())), } ), spec=client.V1NodeSpec( unschedulable=True, taints=[ client.V1Taint( key='node.kubernetes.io/unschedulable', effect='NoSchedule', time_added=(now - datetime.timedelta(days=4)), ) ] )), ], 'all_namespaces': [ client.V1Namespace(metadata=client.V1ObjectMeta( name='ns-1', annotations={ 'annotation-1': 'bla', })), client.V1Namespace(metadata=client.V1ObjectMeta( name='ns-2', annotations={ 'annotation-2': 'blub', })) ], 'all_pods': [ client.V1Pod( metadata=client.V1ObjectMeta(namespace='ns-1', name='pod-1', annotations={ 'annotation-3': '123', }), spec=client.V1PodSpec(node_name='node-5', containers=[])), client.V1Pod( metadata=client.V1ObjectMeta(namespace='ns-2', name='pod-2', annotations={ 'annotation-4': '456', }), spec=client.V1PodSpec(node_name='node-6', containers=[])), client.V1Pod( metadata=client.V1ObjectMeta(namespace='ns-2', name='pod-3', annotations={ 'annotation-5': '789', }), spec=client.V1PodSpec(node_name='node-7', containers=[])), ], 'args': args, } expected_result = { 'cordon': { 'nodes': ['node-1', 'node-2', 'node-4'], 'affected_pods': [] }, 'notify': { 'nodes': ['node-5', 'node-6'], 'affected_pods': [ { 'namespace': 'ns-1', 'name': 'pod-1', 'annotations': { 'annotation-1': 'bla', 'annotation-3': '123', }, 'eviction_time': '2 days from now', }, { 'namespace': 'ns-2', 'name': 'pod-2', 'annotations': { 'annotation-2': 'blub', 'annotation-4': '456', }, 'eviction_time': '2 days from now', }, ] }, 'drain': { 'nodes': ['node-7'], 'affected_pods': [ { 'namespace': 'ns-2', 'name': 'pod-3', 'annotations': { 'annotation-2': 'blub', 'annotation-5': '789', }, 'eviction_time': None, }, ] }, } self.assertEqual(expected_result, main.generate_action_plan(**mock_input))
def update_node(): data = json.loads(request.get_data().decode('utf-8')) print("update_node接收到的数据是{}".format(data)) name = handle_input(data.get("node_name")) action = handle_input(data.get("action")) node = get_node_by_name(name) if node == None: return jsonify({"error": "找不到此node信息"}) if action == "add_taint": print("正在添加node污点") effect = handle_input(data.get('taint_effect')) key = handle_input(data.get('taint_key')) value = handle_input(data.get('taint_value')) # print(type(node.spec.taints)) if node.spec.taints == None: node.spec.taints = [] taint = client.V1Taint(effect=effect, key=key, value=value) node.spec.taints.append(taint) # print(node.spec.taints) elif action == "delete_taint": print("正在删除node污点") effect = handle_input(data.get('taint_effect')) key = handle_input(data.get('taint_key')) value = handle_input(data.get('taint_value')) # print(key,value) # print(type(node.spec.taints)) if node.spec.taints == None: return jsonify({"error": "taint列表为空"}) # 查找元素 i = -1 taint_len = len(node.spec.taints) has_taint = False for taint in node.spec.taints: i = i + 1 # print(taint) if effect == taint.effect and key == taint.key and value == taint.value: has_taint = True break #查找元素 if not has_taint: return jsonify({"error": "没有此taint"}) else: node.spec.taints.pop(i) # print(node.spec.taints) elif action == "update_taint": print("正在更新node污点") old_effect = handle_input(data.get('old_taint_effect')) old_key = handle_input(data.get('old_taint_key')) old_value = handle_input(data.get('old_taint_value')) new_effect = handle_input(data.get('taint_effect')) new_key = handle_input(data.get('taint_key')) new_value = handle_input(data.get('taint_value')) if node.spec.taints == None: node.spec.taints = [] new_taint = client.V1Taint(effect=new_effect, key=new_key, value=new_value) # print(new_taint) # 思路,找到index,替换 # 查找元素 i = -1 taint_len = len(node.spec.taints) has_taint = False for taint in node.spec.taints: i = i + 1 # print(taint) if old_effect == taint.effect and old_key == taint.key and old_value == taint.value: has_taint = True break #查找元素 if not has_taint: return jsonify({"error": "没有此taint"}) else: node.spec.taints[i] = new_taint # print(node.spec.taints) #增加标签 elif action == "add_labels": current_app.logger.debug("正在执行:{}".format(action)) #{"a":1,"b":2} input_labels = handle_input(data.get('labels')) current_app.logger.debug("接收到的数据:{}".format(input_labels)) if input_labels == None: return simple_error_handle("没有收到labels") labels = node.metadata.labels current_app.logger.debug(type(labels), labels) for k, v in input_labels.items(): labels[k] = v node.metadata.labels = labels elif action == "delete_labels": current_app.logger.debug("正在执行:{}".format(action)) #{"a":1,"b":2} input_labels = handle_input(data.get('labels')) current_app.logger.debug("接收到的数据:{}".format(input_labels)) if input_labels == None: return simple_error_handle("没有收到labels") labels = node.metadata.labels current_app.logger.debug(type(labels), labels) for k, v in input_labels.items(): labels.pop(k) current_app.logger.debug("移除标签后:{}".format(labels)) node.metadata.labels = labels else: return jsonify({"error": "不支持此动作{}".format(action)}) try: if action == "delete_labels": result = client.CoreV1Api().replace_node(name=name, body=node) else: result = client.CoreV1Api().patch_node(name=name, body=node) except ApiException as e: body = json.loads(e.body) msg = { "status": e.status, "reason": e.reason, "message": body['message'] } return jsonify({'error': '更新node失败', "msg": msg}) return jsonify({"ok": "{}成功".format(action)})