示例#1
0
def Teardown(tc):
    if tc.skip:
        return api.types.status.SUCCESS

    req = api.Trigger_CreateExecuteCommandsRequest()
    for node in tc.Nodes:
        api.Trigger_AddNaplesCommand(req, node, "rm -rf /data/upgrade_to_same_firmware_allowed")
    resp = api.Trigger(req)
    try:
        for cmd_resp in resp.commands:
            if cmd_resp.exit_code != 0:
                api.PrintCommandResults(cmd_resp)
                api.Logger.error("Teardown failed %s", cmd_resp.command)
    except:
        api.Logger.error("EXCEPTION occured in Naples command")
        return api.types.status.FAILURE

    req = api.Trigger_CreateExecuteCommandsRequest()
    for n in tc.Nodes:
        cmd = 'curl -k -X DELETE https://' + api.GetNicIntMgmtIP(n) + ':'+utils.GetNaplesMgmtPort()+'/api/v1/naples/rollout/'
        api.Trigger_AddHostCommand(req, n, cmd, timeout=100)
    tc.resp = api.Trigger(req)
    for cmd in tc.resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            return api.types.status.FAILURE

    return api.types.status.SUCCESS
示例#2
0
def Teardown(tc):
    req = api.Trigger_CreateExecuteCommandsRequest()
    for node in tc.Nodes:
        api.Trigger_AddNaplesCommand(
            req, node, "rm -rf /update/upgrade_halt_state_machine")
        api.Trigger_AddNaplesCommand(req, node,
                                     "rm -rf /update/pcieport_upgdata")
        api.Trigger_AddNaplesCommand(req, node,
                                     "rm -rf /update/pciemgr_upgdata")
        api.Trigger_AddNaplesCommand(req, node,
                                     "rm -rf /update/pciemgr_upgrollback")
        api.Trigger_AddNaplesCommand(req, node,
                                     "rm -rf /update/nicmgr_upgstate")
        api.Trigger_AddNaplesCommand(
            req, node, "rm -rf /data/upgrade_to_same_firmware_allowed")
    resp = api.Trigger(req)
    for cmd_resp in resp.commands:
        api.PrintCommandResults(cmd_resp)
        if cmd_resp.exit_code != 0:
            api.Logger.error("Setup failed %s", cmd_resp.command)

    req = api.Trigger_CreateExecuteCommandsRequest()
    for n in tc.Nodes:
        cmd = 'curl -k -X DELETE https://' + api.GetNicIntMgmtIP(
            n) + ':8888/api/v1/naples/rollout/'
        api.Trigger_AddHostCommand(req, n, cmd)
    tc.resp = api.Trigger(req)
    for cmd in tc.resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            return api.types.status.FAILURE
    return api.types.status.SUCCESS
示例#3
0
def Verify(tc):
    api.Logger.info(f"Sleeping for {tc.args.sleep} ")
    time.sleep(tc.args.sleep)

    if tc.resp is None:
        api.Logger.error("No response for Naples Upgrade POST request")
        return api.types.status.FAILURE

    for cmd in tc.resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            return api.types.status.FAILURE

    req = api.Trigger_CreateExecuteCommandsRequest()
    for n in tc.Nodes:
        cmd = 'curl -k https://' + api.GetNicIntMgmtIP(
            n) + ':8888/api/v1/naples/rollout/'
        api.Trigger_AddHostCommand(req, n, cmd)
    tc.resp = api.Trigger(req)
    for cmd in tc.resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            api.Logger.info("cmd returned failure")
            return api.types.status.FAILURE
        resp = json.loads(cmd.stdout)
        try:
            for item in resp['Status']['status']:
                if not item['Op'] == 4:
                    api.Logger.info("opcode is bad")
                    return api.types.status.FAILURE
                if "fail" in tc.iterators.option:
                    if not item['opstatus'] == 'failure':
                        api.Logger.info("opstatus is bad")
                        return api.types.status.FAILURE
                    if tc.iterators.option not in item['Message']:
                        api.Logger.info("message is bad")
                        return api.types.status.FAILURE
                else:
                    if not item['opstatus'] == 'success':
                        api.Logger.info("opstatus is bad")
                        return api.types.status.FAILURE
        except:
            api.Logger.info("resp: ", json.dumps(resp, indent=1))

    ping.TestTerminateBackgroundPing(tc, tc.pkt_size)
    ping_loss_duration = ping.GetMaxPktLossDuration(tc, tc.interval)
    api.Logger.info(f"Traffic dropped for {ping_loss_duration} sec")

    if enable_ssh.Main(None) != api.types.status.SUCCESS:
        api.Logger.error("Enabling SSH failed after upgrade")
        return api.types.status.FAILURE

    if upgrade_utils.VerifyUpgLog(tc.Nodes, tc.GetLogsDir()):
        api.Logger.error("Failed to verify the upgrade logs")
        return api.types.status.FAILURE
    if verify_connectivity(tc) != api.types.status.SUCCESS:
        api.Logger.error("Post trigger connectivity test failed.")
        return api.types.status.FAILURE

    return api.types.status.SUCCESS
示例#4
0
def Verify(tc):
    time.sleep(tc.args.sleep)

    if tc.resp is None:
        return api.types.status.FAILURE

    for cmd in tc.resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code != 0:
            return api.types.status.FAILURE

    req = api.Trigger_CreateExecuteCommandsRequest()
    for n in tc.Nodes:
        cmd = 'curl -k https://' + api.GetNicIntMgmtIP(
            n) + ':8888/api/v1/naples/rollout/'
        api.Trigger_AddHostCommand(req, n, cmd)
    tc.resp = api.Trigger(req)

    for cmd in tc.resp.commands:
        api.PrintCommandResults(cmd)

    if  netagent_cfg_api.switch_profile(push_base_profile=True) != \
        api.types.status.SUCCESS:
        api.Logger.warn("Failed to push base profile")
        return api.types.status.FAILURE

    if netagent_cfg_api.PushBaseConfig(ignore_error = False) != \
       api.types.status.SUCCESS:
        api.Logger.info("policy push failed")
        return api.types.status.FAILURE

    for cmd in tc.resp.commands:
        if cmd.exit_code != 0:
            api.Logger.info("cmd returned failure")
            return api.types.status.FAILURE
        if arping.ArPing(tc) != api.types.status.SUCCESS:
            api.Logger.info("arping failed on verify")
        if ping.TestPing(tc, 'local_only', 'ipv4',
                         64) != api.types.status.SUCCESS or ping.TestPing(
                             tc, 'remote_only', 'ipv4',
                             64) != api.types.status.SUCCESS:
            api.Logger.info("ping test failed")
            return api.types.status.FAILURE

        resp = json.loads(cmd.stdout)
        try:
            for item in resp['Status']['status']:
                if not item['Op'] == 4:
                    api.Logger.info("opcode is bad")
                    return api.types.status.FAILURE
                else:
                    if not item['opstatus'] == 'success':
                        api.Logger.info("opstatus is bad")
                        return api.types.status.FAILURE
        except:
            api.logger.info("resp : ", json.dumps(resp, indent=4))

    return api.types.status.SUCCESS
示例#5
0
def VerifyMgmtConnectivity(nodes):
    '''
    Verify management connectivity.
    '''
    if api.IsDryrun():
        return api.types.status.SUCCESS

    result = api.types.status.SUCCESS
    req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
    for node in nodes:
        api.Logger.info("Checking connectivity to Naples Mgmt IP: %s" %
                        api.GetNicIntMgmtIP(node))
        api.Trigger_AddHostCommand(
            req, node, 'ping -c 5 -i 0.2 {}'.format(api.GetNicIntMgmtIP(node)))
    resp = api.Trigger(req)

    if not resp.commands:
        return api.types.status.FAILURE
    for cmd in resp.commands:
        api.PrintCommandResults(cmd)
        if cmd.exit_code:
            result = api.types.status.FAILURE
    return result
示例#6
0
def __get_agent_cfg_nodes(node_names=None, device_names=None):
    agent_node_names = node_names or api.GetNaplesHostnames()
    agent_cfg_nodes = []
    for node_name in agent_node_names:
        assert (api.IsNaplesNode(node_name))
        ip = api.GetNaplesMgmtIpAddress(node_name)
        if not ip:
            assert (0)
        if not device_names:
            device_names = api.GetDeviceNames(node_name)
        for device_name in device_names:
            nic_ip = api.GetNicIntMgmtIP(node_name, device_name)
            agent_cfg_nodes.append(cfg_api.NewCfgNode(node_name, ip, nic_ip))
    return agent_cfg_nodes
示例#7
0
def GetSecurityPolicy(workload=None, node_name=None):
    if not node_name:
        node_name = workload.node_name
    mgmtIp = api.GetNicIntMgmtIP(node_name)
    cmd = 'curl -X GET -k -H "Content-Type:application/json" https://' + mgmtIp + ':8888/api/security/policies/'
    req = api.Trigger_CreateExecuteCommandsRequest()
    result = api.types.status.SUCCESS

    api.Trigger_AddHostCommand(req, node_name, cmd)
    api.Logger.info("Running COMMAND {}".format(cmd))

    resp = api.Trigger(req)

    for cmd in resp.commands:
        api.Logger.info("CMD = {}", cmd)
        if cmd.exit_code != 0:
            return None
    return cmd.stdout
示例#8
0
def GetNaplesMgmtIP(node, device=None):
    return api.GetNicIntMgmtIP(node, device)
示例#9
0
def CopyMemStatsCheckTool(nodes=None):
    if not nodes:
        nodes = api.GetNaplesHostnames()

    for node in nodes:
        # Get the memory stats check tool
        wget_cmd = "wget http://www.pixelbeat.org/scripts/ps_mem.py -O {}".format(MemStatsCheckToolHostPath)
        status, resp = runHostCmd(node, wget_cmd)

        if status != api.types.status.SUCCESS:
            api.Logger.error("Failed to get ps_mem.py tool to naples host: %s"%node)
            return api.types.status.FAILURE

        copy_cmd = "chmod +x {} && sshpass -p pen123 scp -o  UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " \
                   "{} root@{}:{}".format(MemStatsCheckToolHostPath, MemStatsCheckToolHostPath, api.GetNicIntMgmtIP(node), MemStatsCheckToolNaplesPath)
        status, resp = runHostCmd(node, copy_cmd)

        if status != api.types.status.SUCCESS:
            api.Logger.error("Failed to copy ps_mem.py tool to naples: %s"%node)
            return api.types.status.FAILURE

    api.Logger.info("Copied MemStats check tool %s successfully to %s"%(MemStatsCheckToolHostPath, nodes))
    return api.types.status.SUCCESS
示例#10
0
def CheckRolloutStatus(tc):
    retries = 0
    start_ts = datetime.now()
    result = api.types.status.FAILURE
    status_found = False
    while retries < 100:
        api.Logger.info("------Issuing Rollout get %s retry------"%retries)
        misc_utils.Sleep(2)
        retries += 1
        # get rollout status
        req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
        for n in tc.Nodes:
            cmd = 'curl -k https://' + api.GetNicIntMgmtIP(n) + ':'+utils.GetNaplesMgmtPort()+'/api/v1/naples/rollout/'
            api.Trigger_AddHostCommand(req, n, cmd)
            api.Logger.info("Sending rollout status get request: %s"%(cmd))
        tc.resp = api.Trigger(req)

        try:
            for cmd in tc.resp.commands:
                api.PrintCommandResults(cmd)
        except Exception as e:
            api.Logger.error(f"Exception occured in sending rollout status get.{e}")
            continue

        for cmd in tc.resp.commands:
            if cmd.exit_code != 0:
                api.Logger.info("Rollout status get request returned failure")
                continue
            resp = json.loads(cmd.stdout)
            try:
                for item in resp['Status']['status']:
                    status_found = True
                    if not item['Op'] == 4:
                        api.Logger.info("opcode is bad for %s"%cmd.node_name)
                        result = api.types.status.FAILURE
                    if "fail" in tc.iterators.option:
                        if not item['opstatus'] == 'failure':
                            api.Logger.info("opstatus is bad for %s"%cmd.node_name)
                            result = api.types.status.FAILURE
                        if tc.iterators.option not in item['Message']:
                            api.Logger.info("message is bad")
                            result = api.types.status.FAILURE
                    else:
                        if not item['opstatus'] == 'success':
                            api.Logger.info("opstatus(%s) is bad for %s"%(item['opstatus'], cmd.node_name))
                            result = api.types.status.FAILURE
                        else:
                            api.Logger.info("Rollout status is SUCCESS for %s"%cmd.node_name)
                            result = api.types.status.SUCCESS
            except Exception as e:
                api.Logger.error("resp: ", json.dumps(resp, indent=1))
                #api.Logger.error(f"Exception occured in parsing response: {e}")
                result = api.types.status.FAILURE
                continue

        if status_found:
            break

    end_ts = datetime.now()
    # find time elapsed in retrieving rollout status and adjust the wait time for traffic test.
    timedelta = end_ts - start_ts
    time_elapsed = timedelta.days * 24 * 3600 + timedelta.seconds

    if time_elapsed < 100:
        time_elapsed = 100
        misc_utils.Sleep(time_elapsed)

    tc.sleep = (tc.sleep - time_elapsed) if (tc.sleep > time_elapsed) else 10
    return result
示例#11
0
def Trigger(tc):
    if tc.skip:
        return api.types.status.SUCCESS

    req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
    for n in tc.Nodes:
        cmd = 'curl -k -d \'{"kind": "SmartNICRollout","meta": {"name": "test disruptive upgrade","tenant": "tenant-foo"},"spec": {"ops": [{"op": 4,"version": "0.1"}]}}\' -X POST -H "Content-Type:application/json" ' + 'https://' + api.GetNicIntMgmtIP(n) + ':'+utils.GetNaplesMgmtPort()+'/api/v1/naples/rollout/'
        api.Logger.info("Sending rollout request cmd : %s"%cmd)
        api.Trigger_AddHostCommand(req, n, cmd, timeout=30)
    tc.resp = api.Trigger(req)

    api.Logger.info("Rollout request processing complete")
    return api.types.status.SUCCESS
示例#12
0
def Trigger(tc):
    req = api.Trigger_CreateExecuteCommandsRequest()
    for n in tc.Nodes:
        cmd = 'curl -k -d \'{"kind": "SmartNICRollout","meta": {"name": "test disruptive upgrade","tenant": "tenant-foo"},"spec": {"ops": [{"op": 4,"version": "0.1"}]}}\' -X POST -H "Content-Type:application/json" ' + 'https://' + api.GetNicIntMgmtIP(
            n) + ':8888/api/v1/naples/rollout/'
        api.Trigger_AddHostCommand(req, n, cmd)
    tc.resp = api.Trigger(req)

    api.Logger.info("started rollout")
    time.sleep(1)

    api.Logger.info("restarting nodes")
    ret = api.RestartNodes(tc.Nodes)

    return api.types.status.SUCCESS
示例#13
0
def Trigger(tc):
    api.Logger.info("starting ping in background ..")
    ping.TestPing(tc,
                  "all",
                  'ipv4',
                  tc.pkt_size,
                  interval=tc.interval,
                  count=tc.count,
                  background=True,
                  hping3=True)

    req = api.Trigger_CreateExecuteCommandsRequest()
    for n in tc.Nodes:
        cmd = 'curl -k -d \'{"kind": "SmartNICRollout","meta": {"name": "test disruptive upgrade","tenant": "tenant-foo"},"spec": {"ops": [{"op": 4,"version": "0.1"}]}}\' -X POST -H "Content-Type:application/json" ' + 'https://' + api.GetNicIntMgmtIP(
            n) + ':8888/api/v1/naples/rollout/'
        api.Trigger_AddHostCommand(req, n, cmd)
    tc.resp = api.Trigger(req)
    return api.types.status.SUCCESS