def test_check_min_nodes_exist(client, has_conf):
    has_conf.return_value = False
    v1 = MagicMock()

    node1 = create_node_object("old_node")
    taint = k8sClient.V1Taint(effect="NoSchedule",
                              key="node-role.kubernetes.io/master",
                              value=None,
                              time_added=None)
    node1.spec.taints = [taint]
    node1.metadata.creation_timestamp = datetime.datetime.now(
    ) - datetime.timedelta(seconds=1000)

    node2 = create_node_object("new_node")
    taint = k8sClient.V1Taint(effect="NoSchedule",
                              key="node-role.kubernetes.io/master",
                              value=None,
                              time_added=None)
    node2.spec.taints = [taint]
    node2.metadata.creation_timestamp = datetime.datetime.now(
    ) - datetime.timedelta(seconds=10)

    response = k8sClient.V1NodeList(items=[node1, node2])
    v1.list_node_with_http_info.return_value = response
    client.CoreV1Api.return_value = v1
    client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])

    resp = check_min_nodes_exist(k8s_label_selector="some_selector",
                                 min_limit=2)
    assert True == resp

    resp = check_min_nodes_exist(k8s_label_selector="some_selector",
                                 min_limit=5)
    assert False == resp
def test_attach_sq_to_instance_by_tag(gks, client, has_conf, boto_client):
    has_conf.return_value = False
    gks.return_value = {
        'url': 'fake_url.com',
        'token': 'fake_token_towhatever',
        'SLACK_CHANNEL': 'chaos_fanout',
        'SLACK_TOKEN': 'sometoken'
    }
    v1 = MagicMock()

    taint1 = k8sClient.V1Taint(effect="NoSchedule",
                               key="node-role.kubernetes.io/master",
                               value=None,
                               time_added=None)
    taint2 = k8sClient.V1Taint(effect="NoSchedule",
                               key="dedicated",
                               value="spot",
                               time_added=None)

    ignore_list = [taint1, taint2]

    node1 = create_node_object("node1")

    node2 = create_node_object("tainted_node_ignore")
    taint = k8sClient.V1Taint(effect="NoSchedule",
                              key="dedicated",
                              time_added=None,
                              value="spot")
    node2.spec.taints = [taint]

    response = k8sClient.V1NodeList(items=[node1, node2])
    v1.list_node_with_http_info.return_value = response
    client.CoreV1Api.return_value = v1
    client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])

    client = MagicMock()
    boto_client.client.return_value = client
    boto_client.resource.return_value = client
    network_interface = MagicMock()

    instance = MagicMock()
    instance.security_groups = [{"GroupId": "some_testsgid"}]
    instance.network_interfaces = [network_interface]

    client.instances.filter.return_value = [instance]

    client.describe_security_groups.return_value = {
        'SecurityGroups': [{
            'GroupId': "i_testsgid",
        }]
    }
    config = create_config_with_taint_ignore()
    retval = attach_sq_to_instance_by_tag(tag_name="under_chaostest",
                                          sg_name="chaos_test_sg",
                                          configuration=config)

    assert retval is not None
    network_interface.modify_attribute.assert_called_with(
        Groups=['i_testsgid'])
Exemple #3
0
def remove_taint_from_node(
    label_selector: str = None,
    key: str = None,
    value: str = None,
    effect: str = None,
    secrets: Secrets = None,
) -> bool:
    """
    remove taint from nodes by label.As rollback

    """
    # To avoid implementation of non default constructor in tests
    taint_to_remove = client.V1Taint(key="", effect="", value="")
    taint_to_remove.key = key
    taint_to_remove.effect = effect
    taint_to_remove.value = value

    items, k8s_pai_v1 = get_node_list(label_selector, secrets)

    for node in items:
        try:
            logger.warning("Remove taint from node :" + node.metadata.name)
            if node.spec is not None and node.spec is not None and node.spec.taints is not None:
                existing_taints = node.spec.taints
                patch = generate_patch_for_taint_deletion(
                    existing_taints, taint_to_remove)
                k8s_pai_v1.patch_node(node.metadata.name, patch)
        except ApiException as x:
            raise FailedActivity("Un tainting node failed: {}".format(x.body))
    return True
Exemple #4
0
def taint_nodes_by_label(label_selector: str = None,
                         key: str = None,
                         value: str = None,
                         effect: str = None,
                         secrets: Secrets = None) -> bool:
    """
    taint nodes by label.

    """
    # To avoid implementation of non default constructor in tests
    new_taint = client.V1Taint(key="", effect="", value="")
    new_taint.key = key
    new_taint.effect = effect
    new_taint.value = value

    items, k8s_api_v1 = get_node_list(label_selector, secrets)

    for node in items:
        try:
            logger.warning("Taint node :" + node.metadata.name)
            existing_taints = []
            if node.spec is not None and node.spec is not None and node.spec.taints is not None:
                existing_taints = node.spec.taints
            body_patch = generate_patch_for_taint(existing_taints, new_taint)
            k8s_api_v1.patch_node(node.metadata.name, body_patch)
        except ApiException as x:
            raise FailedActivity("tainting node failed: {}".format(x.body))
    return True
Exemple #5
0
def taint_node(node_name):
    """
    Taint a kubernetes node to avoid new pods being scheduled on it
    """

    try:
        config.load_incluster_config()
    except config.ConfigException:
        try:
            config.load_kube_config()
        except config.ConfigException:
            raise Exception("Could not configure kubernetes python client")

    configuration = client.Configuration()
    # create an instance of the API class
    k8s_api = client.CoreV1Api(client.ApiClient(configuration))
    logger.info("Adding taint to k8s node {}...".format(node_name))
    try:
        taint = client.V1Taint(effect='NoSchedule', key='eks-rolling-update')
        api_call_body = client.V1Node(spec=client.V1NodeSpec(taints=[taint]))
        if not app_config['DRY_RUN']:
            k8s_api.patch_node(node_name, api_call_body)
        else:
            k8s_api.patch_node(node_name, api_call_body, dry_run=True)
        logger.info("Added taint to the node")
    except ApiException as e:
        logger.info("Exception when calling CoreV1Api->patch_node: {}".format(e))
Exemple #6
0
def get_tainted_nodes(key: str = None,
                      value: str = None,
                      effect: str = None,
                      secrets: Secrets = None):
    """
    Get nodes that are tainted with specific taint.
    :param key: Taint key
    :param value: Taint value
    :param effect: Taint effect
    :param secrets: chaostoolkit will inject secrets
    :return: array of nodes with specific taint
    """
    all_nodes, k8s_api_v1 = get_active_nodes(label_selector="",
                                             taints_ignore_list=None,
                                             secrets=secrets)
    taint_to_find = client.V1Taint(effect=effect,
                                   key=key,
                                   time_added=None,
                                   value=value)
    retval = []
    for node in all_nodes.items:
        if node.spec is not None and node.spec.taints is not None:
            for taint in node.spec.taints:
                if is_equal_V1Taint(taint, taint_to_find):
                    retval.append(node)
    return retval
def test_generate_patch_for_taint_already_exists():
    taint1 = k8sClient.V1Taint(effect="NoSchedule",
                               key="com.wixpress.somekey",
                               time_added=None,
                               value="ssr")
    taint2 = k8sClient.V1Taint(effect="noExecute",
                               key="com.wixpress.somekey",
                               time_added=None,
                               value="dbmng")
    taint_new = k8sClient.V1Taint(effect="noExecute",
                                  key="com.wixpress.somekey",
                                  time_added=None,
                                  value="dbmng")
    existing_taints = [taint1, taint2]

    patch = generate_patch_for_taint(existing_taints, taint_new)
    assert len(patch['spec']['taints']) is 2
def test_get_non_tainted_nodes_filtered(cl, client, has_conf):
    has_conf.return_value = False
    v1 = MagicMock()

    taint1 = k8sClient.V1Taint(effect="NoSchedule",
                               key="node-role.kubernetes.io/master",
                               value=None,
                               time_added=None)
    taint2 = k8sClient.V1Taint(effect="NoSchedule",
                               key="dedicated",
                               value="spot",
                               time_added=None)

    ignore_list = [taint1, taint2]

    node1 = create_node_object("tainted_node")
    taint = k8sClient.V1Taint(effect="NoSchedule",
                              key="faketaint",
                              time_added=None,
                              value=None)
    node1.spec.taints = [taint]

    node2 = create_node_object("tainted_node_ignore")
    taint = k8sClient.V1Taint(effect="NoSchedule",
                              key="dedicated",
                              time_added=None,
                              value="spot")
    node2.spec.taints = [taint]

    node3 = create_node_object("not_tainted")

    response = k8sClient.V1NodeList(items=[node1, node2, node3])
    v1.list_node_with_http_info.return_value = response
    client.CoreV1Api.return_value = v1
    client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])

    resp, v1 = get_active_nodes(taints_ignore_list=ignore_list)
    assert 2 == len(resp.items)
def test_get_tainted_nodes(cl, client, has_conf):
    has_conf.return_value = False

    def replace_k8s_taint(effect, key, time_added, value):
        return k8sClient.V1Taint(effect=effect,
                                 key=key,
                                 time_added=time_added,
                                 value=value)

    v1 = MagicMock()

    node1 = create_node_object("tainted_node")
    taint = k8sClient.V1Taint(effect="NoSchedule",
                              key="dedicated",
                              time_added=None,
                              value="special")
    node1.spec.taints = [taint]

    node2 = create_node_object("tainted_node_ignore")
    taint = k8sClient.V1Taint(effect="NoSchedule",
                              key="dedicated",
                              time_added=None,
                              value="spot")
    node2.spec.taints = [taint]

    node3 = create_node_object("not_tainted")

    response = k8sClient.V1NodeList(items=[node1, node2, node3])
    v1.list_node_with_http_info.return_value = response
    client.CoreV1Api.return_value = v1
    client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])
    client.V1Taint.return_value = replace_k8s_taint
    resp = get_tainted_nodes(key="dedicated",
                             value="special",
                             effect="NoSchedule")

    assert 1 == len(resp)
    assert resp[0].metadata.name == node1.metadata.name
def load_taint_list_from_dict(list_of_objects):
    """
    Convert list of dictionaries loaded from configuration to V1Taint objects
    :param list_of_objects: array of objects. Usually loaded from configuration
    :return: array of V1Taint objects
    """
    retval = []
    for obj in list_of_objects:
        taint = client.V1Taint(effect=obj.get("effect"),
                               value=obj.get("value"),
                               key=obj.get("key"),
                               time_added=obj.get("time_added"))
        retval.append(taint)
    return retval
Exemple #11
0
def taint_node(node, key, value, effect='NoSchedule'):
    node_name = node.metadata.name
    taints = node.spec.taints or []
    assert not any(taint.key == key for taint in taints)
    taint = client.V1Taint(
        key=key,
        effect=effect,
        time_added=datetime.now(tz=tzutc()),
    )
    taints.append(taint)
    patch = {'spec': {'taints': taints}}
    v1 = client.CoreV1Api()
    rtn = v1.patch_node(node_name, patch)
    return rtn
def test_tag_random_node_aws(clientApi, has_conf, boto_client):
    has_conf.return_value = False
    v1 = MagicMock()

    node1 = create_node_object("node1")

    node2 = create_node_object("tainted_node_ignore")
    taint = k8sClient.V1Taint(effect="NoSchedule",
                              key="dedicated",
                              time_added=None,
                              value="spot")
    node2.spec.taints = [taint]

    response = k8sClient.V1NodeList(items=[node1, node2])
    v1.list_node_with_http_info.return_value = response
    clientApi.return_value = v1

    client = MagicMock()
    boto_client.client.return_value = client

    client.describe_instances.return_value = {
        'Reservations': [{
            'Instances': [{
                'InstanceId': "id_1",
                'InstanceLifecycle': 'normal',
                'PrivateDnsName': 'node1'
            }]
        }]
    }
    config = create_config_with_taint_ignore()

    retval, nodename = tag_random_node_aws(k8s_label_selector="label_selector",
                                           secrets=None,
                                           tag_name="test_tag",
                                           configuration=config)

    assert retval == 0
    assert nodename == "node1"
    client.create_tags.assert_called_with(Resources=['id_1'],
                                          Tags=[{
                                              'Key': 'test_tag',
                                              'Value': 'test_tag'
                                          }])
def test_get_non_tainted_nodes(client, has_conf):
    has_conf.return_value = False
    v1 = MagicMock()

    node1 = create_node_object("tainted_node")
    taint = k8sClient.V1Taint(effect="NoSchedule",
                              key="faketaint",
                              time_added=None,
                              value=None)
    node1.spec.taints = [taint]
    node2 = create_node_object("not_tainted_node")

    response = k8sClient.V1NodeList(items=[node1, node2])
    v1.list_node_with_http_info.return_value = response
    client.CoreV1Api.return_value = v1
    client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])

    resp, v1 = get_active_nodes()
    assert len(resp.items) == 2
Exemple #14
0
def taint_node(node_name):
    """
    Taint a kubernetes node to avoid new pods being scheduled on it
    """

    ensure_config_loaded()

    k8s_api = client.CoreV1Api()
    logger.info("Adding taint to k8s node {}...".format(node_name))
    try:
        taint = client.V1Taint(effect='NoSchedule', key='eks-rolling-update')
        api_call_body = client.V1Node(spec=client.V1NodeSpec(taints=[taint]))
        if not app_config['DRY_RUN']:
            k8s_api.patch_node(node_name, api_call_body)
        else:
            k8s_api.patch_node(node_name, api_call_body, dry_run=True)
        logger.info("Added taint to the node")
    except ApiException as e:
        logger.info("Exception when calling CoreV1Api->patch_node: {}".format(e))
def test_taint_nodes_by_label(gks, cl, client, has_conf):
    fake_node_name = "fake_node.com"
    gks.return_value = {
        'url': 'fake_url.com',
        'token': 'fake_token_towhatever',
        'SLACK_CHANNEL': 'chaos_fanout',
        'SLACK_TOKEN': 'sometoken'
    }
    has_conf.return_value = False
    v1 = MagicMock()

    condition = k8sClient.V1NodeCondition(type="Ready", status="True")
    status = k8sClient.V1NodeStatus(conditions=[condition])
    spec = k8sClient.V1NodeSpec(unschedulable=False)
    metadata = k8sClient.V1ObjectMeta(name=fake_node_name,
                                      labels={"label1": "True"})
    node = k8sClient.V1Node(status=status, spec=spec, metadata=metadata)

    response = k8sClient.V1NodeList(items=[node])

    v1.list_node_with_http_info.return_value = response
    v1.patch_node.return_value = node
    client.CoreV1Api.return_value = v1
    client.V1Taint.return_value = k8sClient.V1Taint(key="",
                                                    value="",
                                                    effect="")

    label_selector = 'label_default=true, label1=True'

    taint_nodes_by_label(label_selector=label_selector,
                         key="key1",
                         value="Apps",
                         effect="NoExec")
    assert v1.patch_node.call_count == 1
    args = v1.patch_node.call_args[0]
    assert args[0] == fake_node_name
    assert args[1]['spec']['taints'][0].key == "key1"
    assert args[1]['spec']['taints'][0].effect == "NoExec"
    assert args[1]['spec']['taints'][0].value == "Apps"
def test_is_equal_v1taint_ok():
    taint1 = client.V1Taint(effect="NoSchedule", key="dedicated", value="spot")
    taint2 = client.V1Taint(effect="NoSchedule", key="dedicated", value="spot")
    assert is_equal_V1Taint(taint1, taint2) is True
def test_node_is_tainted_false():
    ignore_list = load_taint_list_from_dict(json.loads(taint_ignore_list_text)["taints-ignore-list"])

    taint = client.V1Taint(effect="NoSchedule", key="somekey", value=None, time_added=None)

    assert node_should_be_ignored_by_taints([taint], ignore_list) is False
def test_node_is_tainted_true():
    ignore_list = load_taint_list_from_dict(json.loads(taint_ignore_list_text)["taints-ignore-list"])
    taint = client.V1Taint(effect="NoSchedule", key="node-role.kubernetes.io/master", value=None, time_added=None)

    assert node_should_be_ignored_by_taints([taint], ignore_list) is True
Exemple #19
0
    def taint_node(self, name, key, value, effect):
        body = client.V1Node(spec=client.V1NodeSpec(
            taints=[client.V1Taint(key=key, value=value, effect=effect)]))

        return self._v1.patch_node(name, body)
 def replace_k8s_taint(effect, key, time_added, value):
     return k8sClient.V1Taint(effect=effect,
                              key=key,
                              time_added=time_added,
                              value=value)
def test_iptables_block_port_no_taint_only(gks, fabric, client, has_conf,
                                           boto_client):
    fabric_api = MagicMock()
    fabric.return_value = fabric_api
    gks.return_value = {
        'url': 'fake_url.com',
        'token': 'fake_token_towhatever',
        'SLACK_CHANNEL': 'chaos_fanout',
        'SLACK_TOKEN': 'sometoken'
    }
    os.environ["SSH_KEY"] = "keytext"
    os.environ["SSH_USER"] = "******"

    has_conf.return_value = False
    v1 = MagicMock()

    taint1 = k8sClient.V1Taint(effect="NoSchedule",
                               key="node-role.kubernetes.io/master",
                               value=None,
                               time_added=None)
    taint2 = k8sClient.V1Taint(effect="NoSchedule",
                               key="dedicated",
                               value="spot",
                               time_added=None)

    ignore_list = [taint1, taint2]

    node1 = create_node_object("node1")

    node2 = create_node_object("tainted_node_ignore")
    taint = k8sClient.V1Taint(effect="NoSchedule",
                              key="dedicated",
                              time_added=None,
                              value="spot")
    node2.spec.taints = [taint]

    response = k8sClient.V1NodeList(items=[node1, node2])
    v1.list_node_with_http_info.return_value = response
    client.CoreV1Api.return_value = v1
    client.V1NodeList.return_value = k8sClient.V1NodeList(items=[])

    client = MagicMock()
    boto_client.client.return_value = client
    boto_client.resource.return_value = client

    instance = MagicMock()
    instance.pivate_ip_address = "test_ip"
    instance.security_groups = [{"GroupId": "some_testsgid"}]

    client.instances.filter.return_value = [instance]

    client.describe_security_groups.return_value = {
        'SecurityGroups': [{
            'GroupId': "i_testsgid",
        }]
    }
    config = create_config_with_taint_ignore()

    retval = iptables_block_port(tag_name="under_chaostest",
                                 port=53,
                                 protocols=["tcp"],
                                 configuration=config)

    assert retval is not None

    text = "iptables -I PREROUTING  -t nat -p {} --dport {} -j DNAT --to-destination 0.0.0.0:1000".format(
        "tcp", 53)

    fabric.sudo.assert_called_with(text)
    def test_generate_action_plan(self):
        mock_input = {
            'all_nodes': [
                client.V1Node(metadata=client.V1ObjectMeta(
                    name='node-1',
                    creation_timestamp=(now - datetime.timedelta(days=30.1)),
                    annotations={}
                ), spec=client.V1NodeSpec(unschedulable=False)),

                client.V1Node(metadata=client.V1ObjectMeta(
                    name='node-2',
                    creation_timestamp=(now - datetime.timedelta(days=60)),
                    annotations={}
                ), spec=client.V1NodeSpec(unschedulable=False)),

                client.V1Node(metadata=client.V1ObjectMeta(
                    name='node-3',
                    creation_timestamp=(now - datetime.timedelta(days=30.2)),
                    annotations={}
                ), spec=client.V1NodeSpec(unschedulable=True)),

                client.V1Node(metadata=client.V1ObjectMeta(
                    name='node-4',
                    creation_timestamp=(now - datetime.timedelta(days=30.3)),
                    annotations={
                        main.annotation('cordoned'): '',
                    }
                ), spec=client.V1NodeSpec(unschedulable=False)),

                client.V1Node(metadata=client.V1ObjectMeta(
                    name='node-4',
                    creation_timestamp=(now - datetime.timedelta(days=30.4)),
                    annotations={
                        main.annotation('cordoned'): '',
                    }
                ), spec=client.V1NodeSpec(
                    unschedulable=True,
                    taints=[
                        client.V1Taint(
                            key='node.kubernetes.io/unschedulable',
                            effect='NoSchedule',
                            time_added=(now - datetime.timedelta(hours=1)),
                        )
                    ]
                )),

                client.V1Node(metadata=client.V1ObjectMeta(
                    name='node-5',
                    creation_timestamp=(now - datetime.timedelta(days=32.5)),
                    annotations={
                        main.annotation('cordoned'): '',
                    }
                ), spec=client.V1NodeSpec(
                    unschedulable=True,
                    taints=[
                        client.V1Taint(
                            key='node.kubernetes.io/unschedulable',
                            effect='NoSchedule',
                            time_added=(now - datetime.timedelta(days=1.2)),
                        )
                    ]
                )),

                client.V1Node(metadata=client.V1ObjectMeta(
                    name='node-6',
                    creation_timestamp=(now - datetime.timedelta(days=35)),
                    annotations={
                        main.annotation('cordoned'): '',
                    }
                ), spec=client.V1NodeSpec(
                    unschedulable=True,
                    taints=[
                        client.V1Taint(
                            key='node.kubernetes.io/unschedulable',
                            effect='NoSchedule',
                            time_added=(now - datetime.timedelta(days=2)),
                        )
                    ]
                )),

                client.V1Node(metadata=client.V1ObjectMeta(
                    name='node-7',
                    creation_timestamp=(now - datetime.timedelta(days=35)),
                    annotations={
                        main.annotation('cordoned'): '',
                        main.annotation('notifications-sent'): str(int((datetime.datetime.utcnow() - datetime.timedelta(days=2.5)).timestamp())),
                    }
                ), spec=client.V1NodeSpec(
                    unschedulable=True,
                    taints=[
                        client.V1Taint(
                            key='node.kubernetes.io/unschedulable',
                            effect='NoSchedule',
                            time_added=(now - datetime.timedelta(days=4)),
                        )
                    ]
                )),
            ],
            'all_namespaces': [
                client.V1Namespace(metadata=client.V1ObjectMeta(
                    name='ns-1',
                    annotations={
                        'annotation-1': 'bla',
                    })),
                client.V1Namespace(metadata=client.V1ObjectMeta(
                    name='ns-2',
                    annotations={
                        'annotation-2': 'blub',
                    }))
            ],
            'all_pods': [
                client.V1Pod(
                    metadata=client.V1ObjectMeta(namespace='ns-1', name='pod-1', annotations={
                        'annotation-3': '123',
                    }),
                    spec=client.V1PodSpec(node_name='node-5', containers=[])),
                client.V1Pod(
                    metadata=client.V1ObjectMeta(namespace='ns-2', name='pod-2', annotations={
                        'annotation-4': '456',
                    }),
                    spec=client.V1PodSpec(node_name='node-6', containers=[])),
                client.V1Pod(
                    metadata=client.V1ObjectMeta(namespace='ns-2', name='pod-3', annotations={
                        'annotation-5': '789',
                    }),
                    spec=client.V1PodSpec(node_name='node-7', containers=[])),
            ],
            'args': args,
        }
        expected_result = {
            'cordon': {
                'nodes': ['node-1', 'node-2', 'node-4'],
                'affected_pods': []
            },
            'notify': {
                'nodes': ['node-5', 'node-6'],
                'affected_pods': [
                    {
                        'namespace': 'ns-1',
                        'name': 'pod-1',
                        'annotations': {
                            'annotation-1': 'bla',
                            'annotation-3': '123',
                        },
                        'eviction_time': '2 days from now',
                    },
                    {
                        'namespace': 'ns-2',
                        'name': 'pod-2',
                        'annotations': {
                            'annotation-2': 'blub',
                            'annotation-4': '456',
                        },
                        'eviction_time': '2 days from now',
                    },
                ]
            },
            'drain': {
                'nodes': ['node-7'],
                'affected_pods': [
                    {
                        'namespace': 'ns-2',
                        'name': 'pod-3',
                        'annotations': {
                            'annotation-2': 'blub',
                            'annotation-5': '789',
                        },
                        'eviction_time': None,
                    },
                ]
            },
        }
        self.assertEqual(expected_result, main.generate_action_plan(**mock_input))
Exemple #23
0
def update_node():
    data = json.loads(request.get_data().decode('utf-8'))
    print("update_node接收到的数据是{}".format(data))
    name = handle_input(data.get("node_name"))
    action = handle_input(data.get("action"))

    node = get_node_by_name(name)
    if node == None:
        return jsonify({"error": "找不到此node信息"})
    if action == "add_taint":
        print("正在添加node污点")
        effect = handle_input(data.get('taint_effect'))
        key = handle_input(data.get('taint_key'))
        value = handle_input(data.get('taint_value'))
        # print(type(node.spec.taints))
        if node.spec.taints == None:
            node.spec.taints = []
        taint = client.V1Taint(effect=effect, key=key, value=value)
        node.spec.taints.append(taint)
        # print(node.spec.taints)
    elif action == "delete_taint":
        print("正在删除node污点")
        effect = handle_input(data.get('taint_effect'))
        key = handle_input(data.get('taint_key'))
        value = handle_input(data.get('taint_value'))
        # print(key,value)
        # print(type(node.spec.taints))
        if node.spec.taints == None:
            return jsonify({"error": "taint列表为空"})
        # 查找元素
        i = -1
        taint_len = len(node.spec.taints)
        has_taint = False
        for taint in node.spec.taints:
            i = i + 1
            # print(taint)
            if effect == taint.effect and key == taint.key and value == taint.value:
                has_taint = True
                break
        #查找元素
        if not has_taint:
            return jsonify({"error": "没有此taint"})
        else:
            node.spec.taints.pop(i)
            # print(node.spec.taints)
    elif action == "update_taint":
        print("正在更新node污点")
        old_effect = handle_input(data.get('old_taint_effect'))
        old_key = handle_input(data.get('old_taint_key'))
        old_value = handle_input(data.get('old_taint_value'))
        new_effect = handle_input(data.get('taint_effect'))
        new_key = handle_input(data.get('taint_key'))
        new_value = handle_input(data.get('taint_value'))

        if node.spec.taints == None:
            node.spec.taints = []
        new_taint = client.V1Taint(effect=new_effect,
                                   key=new_key,
                                   value=new_value)
        # print(new_taint)
        # 思路,找到index,替换
        # 查找元素
        i = -1
        taint_len = len(node.spec.taints)
        has_taint = False
        for taint in node.spec.taints:
            i = i + 1
            # print(taint)
            if old_effect == taint.effect and old_key == taint.key and old_value == taint.value:
                has_taint = True
                break
        #查找元素
        if not has_taint:
            return jsonify({"error": "没有此taint"})
        else:
            node.spec.taints[i] = new_taint
            # print(node.spec.taints)
    #增加标签
    elif action == "add_labels":
        current_app.logger.debug("正在执行:{}".format(action))
        #{"a":1,"b":2}
        input_labels = handle_input(data.get('labels'))
        current_app.logger.debug("接收到的数据:{}".format(input_labels))
        if input_labels == None:
            return simple_error_handle("没有收到labels")
        labels = node.metadata.labels
        current_app.logger.debug(type(labels), labels)
        for k, v in input_labels.items():
            labels[k] = v
        node.metadata.labels = labels
    elif action == "delete_labels":
        current_app.logger.debug("正在执行:{}".format(action))
        #{"a":1,"b":2}
        input_labels = handle_input(data.get('labels'))
        current_app.logger.debug("接收到的数据:{}".format(input_labels))
        if input_labels == None:
            return simple_error_handle("没有收到labels")
        labels = node.metadata.labels
        current_app.logger.debug(type(labels), labels)
        for k, v in input_labels.items():
            labels.pop(k)
        current_app.logger.debug("移除标签后:{}".format(labels))
        node.metadata.labels = labels
    else:
        return jsonify({"error": "不支持此动作{}".format(action)})
    try:
        if action == "delete_labels":
            result = client.CoreV1Api().replace_node(name=name, body=node)
        else:
            result = client.CoreV1Api().patch_node(name=name, body=node)
    except ApiException as e:
        body = json.loads(e.body)
        msg = {
            "status": e.status,
            "reason": e.reason,
            "message": body['message']
        }
        return jsonify({'error': '更新node失败', "msg": msg})

    return jsonify({"ok": "{}成功".format(action)})