Пример #1
0
def Setup(tc):
    tc.Nodes    = api.GetNaplesHostnames()
    tc.uuidMap  = api.GetNaplesNodeUuidMap()
    tc.wl       = None
    tc.old_node = None
    tc.new_node = None 
    return api.types.status.SUCCESS
Пример #2
0
def Setup(tc):
    tc.Nodes = api.GetNaplesHostnames()
    tc.AllNodes = api.GetWorkloadNodeHostnames()
    tc.uuidMap = api.GetNaplesNodeUuidMap()
    tc.new_node = None
    tc.old_node = None
    tc.vm_non_dsc_to_dsc = False
    tc.vm_dsc_to_dsc = False
    tc.move_info = []
    if tc.args.vm_type == 'non_dsc_to_dsc':
        tc.vm_non_dsc_to_dsc = True
    else:
        tc.vm_dsc_to_dsc = True
    tc.num_moves = int(getattr(tc.args, "num_moves", 1))

    getNonNaplesNodes(tc)
    if arping.ArPing(tc) != api.types.status.SUCCESS:
        api.Logger.info("arping failed on setup")
    if ping.TestPing(tc, 'local_only', 'ipv4',
                     64) != api.types.status.SUCCESS or ping.TestPing(
                         tc, 'remote_only', 'ipv4',
                         64) != api.types.status.SUCCESS:
        api.Logger.info("ping test failed on setup")
        return api.types.status.FAILURE

    #Start Fuz
    ret = start_fuz(tc)
    if ret != api.types.status.SUCCESS:
        api.Logger.error("Fuz start failed")
        return api.types.status.FAILURE

    return api.types.status.SUCCESS
Пример #3
0
def PrepareWorkloadVMotion(tc, candidateWorkloads):
    tc.vmotion_cntxt = parser.Dict2Object({})

    tc.vmotion_cntxt.OrigHome = {}
    tc.vmotion_cntxt.CurrentHome = {}
    tc.vmotion_cntxt.MoveRequest = {}
    tc.vmotion_cntxt.UUIDMap = api.GetNaplesNodeUuidMap()
    tc.vmotion_cntxt.TimeProfile = []

    for wl in candidateWorkloads:
        other_hosts = [
            _n.Name() for _n in api.GetNodes() if _n.Name() != wl.node_name
        ]
        if other_hosts:
            dest_host = other_hosts[0]
        else:
            api.Logger.warn("No alternative host found to move %s" %
                            wl.workload_name)
            continue
        tc.vmotion_cntxt.OrigHome[wl] = wl.node_name
        tc.vmotion_cntxt.CurrentHome[wl] = wl.node_name

        if dest_host not in tc.vmotion_cntxt.MoveRequest:
            tc.vmotion_cntxt.MoveRequest[dest_host] = []
        tc.vmotion_cntxt.MoveRequest[dest_host].append(wl)
        api.Logger.info("Preparing to move %s to %s" %
                        (wl.workload_name, dest_host))
    return
Пример #4
0
def UpdateNodeUuidEndpoints(objects):
    #allocate pvlan pair for dvs
    if api.GetConfigNicMode() == 'hostpin_dvs':
        pvlan_start = api.GetPVlansStart()
        pvlan_end = api.GetPVlansEnd()
        index = 0
        for obj in objects:
            obj.spec.primary_vlan = pvlan_start + index
            obj.spec.secondary_vlan = pvlan_start + index + 1
            obj.spec.useg_vlan = obj.spec.primary_vlan
            index = index + 2
            if index > pvlan_end:
                assert (0)

    agent_uuid_map = api.GetNaplesNodeUuidMap()
    for ep in objects:
        node_name = getattr(ep.spec, "_node_name", None)
        if not node_name:
            node_name = ep.spec.node_uuid
        assert (node_name)
        if api.IsNaplesNode(node_name):
            ep.spec.node_uuid = formatMac(GetNaplesUUID(node_name))
        else:
            ep.spec.node_uuid = agent_uuid_map[node_name]
        ep.spec._node_name = node_name
Пример #5
0
def Setup(tc):
    vm_threads = []
    node_list = []
    node = getattr(tc.args, "node", None)
    if node:
        node_list.append(node)
    else:
        '''
        add all nodes in the topo
        '''
        nodes = api.GetNodes()
        for node in nodes:
            node_list.append(node.Name())
    tc.uuidMap = api.GetNaplesNodeUuidMap()
    for node in node_list:
        (wls, new_node) = getWorkloadsToRemove(tc, node)
        for wl in wls:
            api.Logger.info("Moving wl {} from node {} to node {}".format(
                wl.workload_name, wl.node_name, new_node))
            vm_thread = threading.Thread(target=triggerVmotion,
                                         args=(
                                             tc,
                                             wl,
                                             new_node,
                                         ))
            vm_threads.append(vm_thread)
            vm_thread.start()
            if (api.IsNaplesNode(new_node)):
                create_ep_info(tc, wl, new_node, "START", node)
        for vm_thread in vm_threads:
            vm_thread.join()
        for wl in wls:
            if (api.IsNaplesNode(node)):
                delete_ep_info(tc, wl, node)
    return api.types.status.SUCCESS
Пример #6
0
def Setup(tc):
    tc.Nodes    = api.GetNaplesHostnames()
    tc.uuidMap  = api.GetNaplesNodeUuidMap()
    tc.old_node = None
    tc.new_node = None
    tc.skip_security_prof = getattr(tc.args, "skip_security_prof", False)

    tc.client, tc.server, tc.naples = None, None, None

    if tc.args.type == 'local_only':
        tc.workload_pairs = api.GetLocalWorkloadPairs(naples=True)
    else:
        tc.workload_pairs = api.GetRemoteWorkloadPairs()

    if len(tc.workload_pairs) == 0:
        api.Logger.error("Failed to get client server pair")
        return api.types.status.FAIL

    # pick a node to run traffic test
    ret = chooseNaples(tc, tc.workload_pairs[0])
    if ret != api.types.status.SUCCESS:
        return ret

    # pick a new node to move one of the vms to
    ret = vm_utils.pick_new_node(tc)
    if ret != api.types.status.SUCCESS:
        return ret

    if not addPktFltrRuleOnEp(tc):
        api.Logger.error("Failed to add packet filter on EP")
        return api.types.status.FAIL

    return api.types.status.SUCCESS
Пример #7
0
def Setup(tc):
    tc.Nodes = api.GetNaplesHostnames()
    tc.uuidMap = api.GetNaplesNodeUuidMap()
    tc.old_node = None
    tc.new_node = None
    tc.skip_security_prof = getattr(tc.args, "skip_security_prof", False)

    if tc.args.type == 'local_only':
        tc.workload_pairs = api.GetLocalWorkloadPairs()
    else:
        tc.workload_pairs = api.GetRemoteWorkloadPairs()
    w1, w2 = tc.workload_pairs[0]
    tc.old_node = w1.node_name if w1.IsNaples() else w2.node_name
    tc.wl = w1

    # remove the node being used to pick wl for traffic
    # pick the next node in the naples nodes
    naples_nodes = tc.Nodes[:]
    naples_nodes.remove(w1.node_name)
    assert (len(naples_nodes) >= 1)
    new_node = naples_nodes[0]
    tc.new_node = new_node
    move_info = vm_utils.MoveInfo()
    move_info.new_node = new_node
    move_info.wl = w1
    move_info.old_node = w1.node_name
    tc.move_info.append(move_info)
    return api.types.status.SUCCESS
Пример #8
0
def Main(args):
    defs.DOL_PATH = "/iota/"
    defs.TEST_TYPE = "IOTA"
    EzAccessStore.SetTestbedSpec(api.GetTestbedSpec())
    for node, cfgyml in vars(args.spec).items():
        api.Logger.info(f"Generating Configuration for Spec {cfgyml}")
        cfgspec = parser.ParseFile('test/apulu/config/cfg/', f'{cfgyml}')

        naples_uuid_map = api.GetNaplesNodeUuidMap()
        EzAccessStoreInit(node)
        EzAccessStore.SetUuidMap(naples_uuid_map)
        EzAccessStoreClient[node].SetUnderlayIPs(api.GetNicUnderlayIPs(node))
        NodeClient.CreateNode(node, cfgspec, api.GetNicMgmtIP(node))

    if not __generate_rmappings_from_lmappings():
        api.Logger.error("Failed to convert local mappings to remote mappings")
        return api.types.status.FAILURE

    # Update static NextHop objects with the mac-addresses of the peer's interfaces
    # This is temporary until the dynamic underlay NH stitching support comes in soon.
    __update_nexthops_from_uplink_info()
    __update_host_interfaces()

    naplesNodes = api.GetNaplesHostnames()
    MemStatsClient.InitNodesForMemUsageStats(naplesNodes)
    for node in naplesNodes:
        storeClient = EzAccessStoreClient[node]
        if storeClient.IsDeviceLearningEnabled():
            if not learn_utils.SetDeviceLearnTimeout(storeClient.GetDevice().GetLearnAgeTimeout()):
                return api.types.status.FAILURE
            if not learn_utils.ClearLearnData():
                return api.types.status.FAILURE

    return api.types.status.SUCCESS
Пример #9
0
def Setup(tc):
    tc.move_info = []
    tc.uuidMap = api.GetNaplesNodeUuidMap()
    tc.num_moves = int(getattr(tc.args, "num_moves", 1))
    tc.node_to_move = getattr(tc.args, "", "client")
    tc.local_or_remote = getattr(tc.args, "", "local")
    update_sgpolicy('dns', True)
    return api.types.status.SUCCESS
Пример #10
0
def UpdateCurrentHome(tc):

    # Following processing is assuming all-or-none for success. Will be reviewed if otherwise
    if tc.vmotion_resp == api.types.status.SUCCESS:
        api.Logger.debug("Update current home of workloads post vMotion")
        for dest_host, workloads in tc.vmotion_cntxt.MoveRequest.items():
            for wl in workloads:
                tc.vmotion_cntxt.CurrentHome[wl] = dest_host
        tc.vmotion_cntxt.UUIDMap = api.GetNaplesNodeUuidMap()
    return
Пример #11
0
def Setup(tc):
    tc.move_info = []
    tc.uuidMap = api.GetNaplesNodeUuidMap()
    tc.num_moves = int(getattr(tc.args, "num_moves", 1))
    tc.node_to_move = getattr(tc.args, "", "client")
    tc.local_or_remote = getattr(tc.args, "", "local")
    update_app('ftp', '120s')
    if tc.iterators.mode == 'active':
        update_sgpolicy('ftp', True)
    return api.types.status.SUCCESS
Пример #12
0
def Setup(tc):
    tc.move_info = []
    tc.server = ''
    tc.client = ''
    tc.uuidMap = api.GetNaplesNodeUuidMap()
    tc.num_moves = int(getattr(tc.args, "num_moves", 2))
    tc.num_b2b_moves = int(getattr(tc.args, "b2b_moves", 2))
    tc.node_to_move = getattr(tc.args, "move", "client")
    tc.local_or_remote = getattr(tc.args, "type", "local")
    # update_sgpolicy('sunrpc_tcp', True)
    return api.types.status.SUCCESS
Пример #13
0
def __generate_config():
    api.Logger.info("Generating Configuration.")
    req = cfg_svc_pb2.GenerateConfigMsg()
    node_uuid_map = api.GetNaplesNodeUuidMap()
    for name, uuid in node_uuid_map.items():
        host = req.hosts.add()
        host.name = name
        host.uuid = uuid

    resp = api.GenerateConfigs(req)
    if resp == None:
        return api.types.status.FAILURE

    api.SetVeniceConfigs(resp.configs)

    return api.types.status.SUCCESS
Пример #14
0
def Setup(tc):
    vm_threads = []
    node_list  = []
    node = getattr(tc.args, "node", None)
    if node:
        node_list.append(node)
    else:
        '''
        add all nodes in the topo
        '''
        nodes = api.GetNodes()
        for node in nodes:
            node_list.append(node.Name())
    tc.Nodes    = api.GetNaplesHostnames()
    tc.AllNodes = api.GetWorkloadNodeHostnames()
    tc.uuidMap  = api.GetNaplesNodeUuidMap()
    tc.move_info         = []
    tc.vm_dsc_to_dsc     = True
    tc.num_moves         = 0

    if hasattr(tc.args, "conntrack"):
        tc.detailed = True
    else:
        tc.detailed = False


    getNonNaplesNodes(tc)
    if arping.ArPing(tc) != api.types.status.SUCCESS:
        api.Logger.info("arping failed on setup")
    if ping.TestPing(tc, 'local_only', 'ipv4', 64) != api.types.status.SUCCESS or ping.TestPing(tc, 'remote_only', 'ipv4', 64) != api.types.status.SUCCESS:
        api.Logger.info("ping test failed on setup")
        return api.types.status.FAILURE

    for node in node_list:
        (wls,new_node) = getWorkloadsToRemove(tc, node)
        tc.num_moves = len(wls)
        vm_utils.update_move_info(tc, wls, False, new_node)

    #Start Fuz
    ret = vm_utils.start_fuz(tc)
    if ret != api.types.status.SUCCESS:
        api.Logger.error("Fuz start failed")
        return api.types.status.FAILURE

    return api.types.status.SUCCESS
Пример #15
0
def Setup(tc):
    tc.Nodes = api.GetNaplesHostnames()
    tc.AllNodes = api.GetWorkloadNodeHostnames()
    tc.uuidMap = api.GetNaplesNodeUuidMap()
    tc.new_node = None
    tc.old_node = None
    tc.vm_non_dsc_to_dsc = False
    tc.vm_dsc_to_dsc = False
    tc.move_info = []
    if tc.args.vm_type == 'non_dsc_to_dsc':
        tc.vm_non_dsc_to_dsc = True
    else:
        tc.vm_dsc_to_dsc = True
    tc.num_moves = int(getattr(tc.args, "num_moves", 1))

    tc.trigger = None
    tc.trigger = getattr(tc.args, "trigger_type", None)
    tc.trigger_on = getattr(tc.args, "trigger_on", 'new')
    '''
    if tc.args.trigger_type  == 'port_flap':
        tc.trigger = 'port_flap'
    elif tc.args.trigger_type == 'mgmt_down':
        tc.trigger = 'mgmt_down'
    elif tc.args.trigger_type == 'delete_ep':
        tc.trigger = 'delete_ep'
    '''

    getNonNaplesNodes(tc)
    #Start Fuz

    if arping.ArPing(tc) != api.types.status.SUCCESS:
        api.Logger.info("arping failed on setup")
    tc1 = ping.TestPing(tc, 'local_only', 'ipv4', 64)
    tc2 = ping.TestPing(tc, 'remote_only', 'ipv4', 64)
    if tc1 != api.types.status.SUCCESS or tc2 != api.types.status.SUCCESS:
        api.Logger.info("ping test failed on setup")
        return api.types.status.FAILURE

    ret = start_fuz(tc, "20s", "20")
    if ret != api.types.status.SUCCESS:
        api.Logger.error("Fuz start failed")
        return api.types.status.FAILURE
    stop_fuz(tc)
    return api.types.status.SUCCESS
Пример #16
0
def Main(step):
    if GlobalOptions.skip_setup:
        #No mode switch required for skeip setup
        return api.types.status.SUCCESS
    req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
    uuidMap = api.GetNaplesNodeUuidMap()
    nodes = api.GetNaplesHostnames()
    for n in nodes:
        # Touch a file to indicate to NMD that the current mode is emulation
        cmd = "touch /data/iota-emulation"
        api.Trigger_AddNaplesCommand(req, n, cmd)
        # Make sure console is enabled
        CreateConfigConsoleNoAuth()
        api.CopyToNaples(n, [NAPLES_CONFIG_SPEC_LOCAL], "")
        cmd = "mv /system-config.json /sysconfig/config0/system-config.json"
        api.Trigger_AddNaplesCommand(req, n, cmd)

        if common.PenctlGetModeStatus(n) != "NETWORK" or common.PenctlGetTransitionPhaseStatus(n) != "VENICE_REGISTRATION_DONE":
            api.Logger.info("Host [{}] is in HOST mode. Initiating mode change.".format(n))
            ret = common.SetNaplesModeOOB_Static(n, "1.1.1.1", "1.1.1.2/24")
            if ret == None:
                return api.types.status.FAILURE

        #hack for now, need to set date
        cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
        api.Trigger_AddNaplesCommand(req, n, cmd)

    resp = api.Trigger(req)

    for cmd in resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            return api.types.status.FAILURE

    num_retries = 60
    reboot_nodes = []
    while nodes:
        req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
        for n in nodes:
            #hack for now, need to set date
            api.Logger.info("Checking Transition phase for node : %s" % n)
            check_state_cmd = "show naples --json"
            time.sleep(30)
            common.AddPenctlCommand(req, n, check_state_cmd)
            api.Trigger_AddNaplesCommand(req, n, "touch /data/no_watchdog")
            resp = api.Trigger(req)
            cmd = resp.commands[0]
            api.PrintCommandResults(cmd)
            if cmd.exit_code != 0:
                return api.types.status.FAILURE
            try:
                out = json.loads(cmd.stdout)
            except:
                api.Logger.error("Penctl output not in Json format {}".format(cmd.stdout))
                return api.types.status.FAILURE
            if not hal_show_utils.IsNaplesForwardingModeClassic(n):
                api.Logger.info("Dataplane already in HOSTPIN mode. Skipping node [{}] for reboot.".format(n))
                reboot_nodes.append(n)
                nodes.remove(n)
            elif out["status"]["transition-phase"] == "VENICE_UNREACHABLE":
                api.Logger.info("Reboot pending on node : %s" % n)
                reboot_nodes.append(n)
                nodes.remove(n)
            elif out["status"]["transition-phase"] == "VENICE_REGISTRATION_DONE":
                api.Logger.info("Node already transitioned : %s" % n)
                nodes.remove(n)
            else:
                api.Logger.info("Reboot not pending on node : %s" % n)

        time.sleep(1)
        num_retries = num_retries - 1
        if num_retries == 0:
            api.Logger.error("Reboot pending state not transitioned complete on naples")
            return api.types.status.FAILURE

    ret = api.RestartNodes(reboot_nodes)
    if ret != api.types.status.SUCCESS:
        api.Logger.error("Node restart failed")
        return api.types.status.FAILURE

    req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
    #enable_sshd = "system enable-sshd"
    #copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub"
    for n in api.GetNaplesHostnames():
        #hack for now, need to set date
        cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
        api.Trigger_AddNaplesCommand(req, n, cmd)
        #common.AddPenctlCommand(req, n, enable_sshd)
        #common.AddPenctlCommand(req, n, copy_key)

    resp = api.Trigger(req)

    for cmd in resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            result = api.types.status.FAILURE

    #Give some time for naples to admit to venice after mode switch
    #This is required for now as Iota is setting time manually to make TLS happy
    time.sleep(30)
    #Check whether naples has switch mode succesfully
    req = api.Trigger_CreateExecuteCommandsRequest(serial = False)
    for n in nodes:
        cmd = "cat /sysconfig/config0/app-start.conf  | grep hostpin"
        api.Trigger_AddNaplesCommand(req, n, cmd)

    resp = api.Trigger(req)
    for cmd in resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            api.Logger.error("Mode switch failed on node : {}".format(cmd.node_name))
            result = api.types.status.FAILURE

    #Verify Agent in right mode.
    req = api.Trigger_CreateExecuteCommandsRequest(serial = False)
    for n in nodes:
        cmd = "curl localhost:8888/api/system/info/"
        api.Trigger_AddNaplesCommand(req, n, cmd)

        # Delete the iota-emulation file created earlied for NMD. Mode change would have passed by now if it had to.
        cmd = "rm -f /data/iota-emulation"
        api.Trigger_AddNaplesCommand(req, n, cmd)

    resp = api.Trigger(req)
    for cmd in resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            api.Logger.error("Agent system get failed : {}".format(cmd.node_name))
            result = api.types.status.FAILURE
        out = None
        try:
            out = json.loads(cmd.stdout)
        except:
            api.Logger.error("Agent System get out failed {}".format(cmd.stdout))
            return api.types.status.FAILURE
        if out["naples-mode"] != "NETWORK_MANAGED_OOB":
            api.Logger.error("Agent not in correct mode: {} {} ".format(cmd.node_name, out["naples-mode"]))
            return api.types.status.FAILURE

    api.Logger.info("Trying to re-enable ssh on naples nodes")
    
    return enable_ssh.Main(None)
Пример #17
0
def Setup(tc):
    '''
    for n in api.GetNaplesHostnames():
        uuid = GetNaplesUUID(n)
        api.SetNaplesNodeUuid(n, uuid)
    '''

    tc.Nodes = api.GetNaplesHostnames()
    tc.AllNodes = api.GetWorkloadNodeHostnames()
    tc.uuidMap = api.GetNaplesNodeUuidMap()
    tc.new_node = None
    tc.old_node = None
    tc.vm_non_dsc_to_dsc = False
    tc.vm_dsc_to_dsc = False
    tc.skip_teardown = False
    tc.move_info = []
    tc.dbg_stats_before = dict()
    tc.dbg_stats_after = dict()
    if tc.args.vm_type == 'non_dsc_to_dsc':
        tc.vm_non_dsc_to_dsc = True
    else:
        tc.vm_dsc_to_dsc = True
    tc.num_moves = int(getattr(tc.args, "num_moves", 1))
    tc.dsc_conn_type = getattr(tc.args, "dsc_con_type", "oob")

    if hasattr(tc.args, "conntrack"):
        tc.detailed = True
    else:
        tc.detailed = False

    if hasattr(tc.args, "skip_teardown"):
        api.Logger.info("Setting tc to skip teardown")
        tc.skip_teardown = True
        #if tc.args.skip_teardown == "True":
        #    tc.skip_teardown = True

    vm_utils.increase_timeout()

    getNonNaplesNodes(tc)
    if arping.ArPing(tc) != api.types.status.SUCCESS:
        api.Logger.info("arping failed on setup")
    if ping.TestPing(tc, 'local_only', 'ipv4',
                     64) != api.types.status.SUCCESS or ping.TestPing(
                         tc, 'remote_only', 'ipv4',
                         64) != api.types.status.SUCCESS:
        api.Logger.info("ping test failed on setup")
        return api.types.status.FAILURE
    '''
    identify workloads to be moved, as we want to run fuz if wl being moved is involved
    '''
    vm_utils.create_move_info(tc, tc.vm_dsc_to_dsc)

    vm_utils.get_vm_dbg_stats(tc)

    #Start Fuz
    ret = vm_utils.start_fuz(tc)
    if ret != api.types.status.SUCCESS:
        api.Logger.error("Fuz start failed")
        return api.types.status.FAILURE

    return api.types.status.SUCCESS
Пример #18
0
def Main(step):
    req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
    venice_ips = api.GetVeniceMgmtIpAddresses()
    assert (venice_ips)
    uuidMap = api.GetNaplesNodeUuidMap()
    for n in api.GetNaplesHostnames():
        #hack for now, need to set date
        cmd = "date -s '{}'".format(
            datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
        api.Trigger_AddNaplesCommand(req, n, cmd)
        api.Logger.info("Running mode switch on node : %s" % n)
        mode_switch_cmd = "update naples --managed-by network --management-network oob --hostname {} --primary-mac {}".format(
            __get_mgmt_nw(), n, uuidMap[n])
        common.AddPenctlCommand(req, n, mode_switch_cmd)

    resp = api.Trigger(req)

    for cmd in resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            return api.types.status.FAILURE

    nodes = api.GetNaplesHostnames()
    num_retries = 60
    while nodes:
        req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
        for n in nodes:
            #hack for now, need to set date
            api.Logger.info("Checking Tranisition phase for node : %s" % n)
            check_state_cmd = "show naples --json"
            common.AddPenctlCommand(req, n, check_state_cmd)
            resp = api.Trigger(req)
            cmd = resp.commands[0]
            api.PrintCommandResults(cmd)
            if cmd.exit_code != 0:
                return api.types.status.FAILURE
            try:
                out = json.loads(cmd.stdout)
            except:
                api.Logger.error("Penctl output not in Json format {}".format(
                    cmd.stdout))
                return api.types.status.FAILURE
            if out["status"]["transition-phase"] == "REBOOT_PENDING":
                api.Logger.info("Reboot pending on node : %s" % n)
                nodes.remove(n)
            else:
                api.Logger.info("Reboot not pending on node : %s" % n)

        time.sleep(1)
        num_retries = num_retries - 1
        if num_retries == 0:
            api.Logger.error(
                "Reboot pending state not transitioned complete on naples")
            return api.types.status.FAILURE

    reboot_nodes = []
    for n in api.GetNaplesHostnames():
        reboot_nodes.append(n)

    ret = api.RestartNodes(reboot_nodes)
    if ret != api.types.status.SUCCESS:
        api.Logger.error("Node restart failed")
        return api.types.status.FAILURE

    req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
    for n in api.GetNaplesHostnames():
        #hack for now, need to set date
        cmd = "date -s '{}'".format(
            datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
        api.Trigger_AddNaplesCommand(req, n, cmd)

    resp = api.Trigger(req)

    for cmd in resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            result = api.types.status.FAILURE

    #Give some time for naples to admit to venice after mode switch
    #This is required for now as Iota is setting time manually to make TLS happy
    time.sleep(30)
    #Check whether naples has switch mode succesfully
    req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
    for n in api.GetNaplesHostnames():
        cmd = "cat /sysconfig/config0/app-start.conf  | grep hostpin"
        api.Trigger_AddNaplesCommand(req, n, cmd)

    resp = api.Trigger(req)
    for cmd in resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            api.Logger.error("Mode switch failed on node : {}".format(
                cmd.node_name))
            result = api.types.status.FAILURE

    #Verify Agent in right mode.
    req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
    for n in api.GetNaplesHostnames():
        cmd = "curl localhost:8888/api/system/info/"
        api.Trigger_AddNaplesCommand(req, n, cmd)

    resp = api.Trigger(req)
    venice_urls = []
    for ip in api.GetVeniceMgmtIpAddresses():
        venice_urls.append(ip + ":9009")
    for cmd in resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            api.Logger.error("Agent system get failed : {}".format(
                cmd.node_name))
            result = api.types.status.FAILURE
        out = None
        try:
            out = json.loads(cmd.stdout)
        except:
            api.Logger.error("Agent System get out failed {}".format(
                cmd.stdout))
            return api.types.status.FAILURE
        if out["naples-mode"] != "NETWORK_MANAGED_OOB":
            api.Logger.error("Agent not in correct mode: {} {} ".format(
                cmd.node_name, out["naples-mode"]))
            return api.types.status.FAILURE
        if set(out["controller-ips"]) != set(venice_urls):
            api.Logger.error("Agent controller IPs don't match: {} {}".format(
                venice_urls, out["controller-ips"]))
            return api.types.status.FAILURE

    return api.types.status.SUCCESS