def Trigger(tc):
tc.cmd_cookies = []
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
common.AddPenctlCommand(req, n, "update time")
common.AddPenctlCommand(req, n, "show time")
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
common.AddPenctlCommand(req, n, "show proc-meminfo")
common.AddPenctlCommand(req, n, "show system-memory-usage")
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
common.AddPenctlCommand(req, n, "system tech-support")
common.AddPenctlCommand(req, n, "show system statistics pb detail")
common.AddPenctlCommand(req, n, "show qos-class queues")
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
common.AddPenctlCommand(req, n, "-v")
common.AddPenctlCommand(req, n, "--version")
common.AddPenctlCommand(req, n, "version")
common.AddPenctlCommand(req, n, "help")
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
tc.cmd_cookies = []
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
common.AddPenctlCommand(req, n, "show firmware-version")
common.AddPenctlCommand(req, n, "show running-firmware")
common.AddPenctlCommand(req, n, "show startup-firmware")
#common.AddPenctlCommand(req, n, "update startup-firmware mainfwa")
common.AddPenctlCommand(req, n, "show startup-firmware")
#common.AddPenctlCommand(req, n, "update startup-firmware mainfwb")
common.AddPenctlCommand(req, n, "show startup-firmware")
#TODO install and save running-firmware variable/verify/etc
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def StartSSH():
api.Logger.info("Starting SSH server on Nodes")
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
enable_sshd = "system enable-sshd"
copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub"
for n in api.GetNaplesHostnames():
#hack for now, need to set date
cmd = "date -s '{}'".format(
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
common.AddPenctlCommand(req, n, enable_sshd)
common.AddPenctlCommand(req, n, copy_key)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
def Create(self, req):
ltep_parent_mac = host_utils.GetMACAddress(self.ltep_node,
self.ltep.parent_interface)
rtep_parent_mac = host_utils.GetMACAddress(self.rtep_node,
self.rtep.parent_interface)
cmd = "update interface --name %s " % self.ltep.interface
cmd += "--gw-mac %s " % rtep_parent_mac
cmd += "--substrate-ip %s " % self.ltep.ip_address
cmd += "--overlay-ip %s " % self.ltep.ip_address
cmd += "--mpls-in %d " % self.label_in
cmd += "--mpls-out %d " % self.label_out
cmd += "--tunnel-dest-ip %s " % self.rtep.ip_address
cmd += "--source-gw %s " % self.gateway_prefix
cmd += "--ingress-bw %d " % self.policer.ingress
cmd += "--egress-bw %d " % self.policer.egress
cmd += "--encap %s " % self.encap
cmd += "--pf-mac %s " % ltep_parent_mac
cmd += "--overlay-mac %s " % self.ltep.mac_address
api.Logger.info("Creating Tunnel %s ==> %s" %
(self.ltep.workload_name, self.rtep.workload_name))
api.Logger.info("- command: %s" % cmd)
common.AddPenctlCommand(req, self.ltep.node_name, cmd)
self.__add_static_arp(req)
return api.types.status.SUCCESS
def Main(step):
if GlobalOptions.skip_setup:
return api.types.status.SUCCESS
naplesHosts = api.GetNaplesHostnames()
assert(len(naplesHosts) != 0)
binary_path = os.path.join(BIN_PATH, testupgapp_utils.UPGRADE_TEST_APP + "_bin",
testupgapp_utils.UPGRADE_TEST_APP + ".bin")
fullpath = os.path.join(api.GetTopDir(), binary_path)
if not os.path.isfile(fullpath):
ALT_BIN_PATH="nic/build/aarch64/iris/capri/out/"
binary_path = os.path.join(ALT_BIN_PATH, testupgapp_utils.UPGRADE_TEST_APP + "_bin",
testupgapp_utils.UPGRADE_TEST_APP + ".bin")
for naplesHost in naplesHosts:
testupgapp_utils.stopTestUpgApp(naplesHost, True)
ret = utils.installBinary(naplesHost, binary_path)
if ret != api.types.status.SUCCESS:
api.Logger.error("Failed in test upgrade app %s copy to Naples" % binary_path)
return ret
req = api.Trigger_CreateExecuteCommandsRequest()
for n in naplesHosts:
pencommon.AddPenctlCommand(req, n, "update time")
api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
# sub_command = getattr(tc.iterators, "subcmd", 'None')
# api.Logger.info("cmd:%s, use_token:%s, sub_cmd:%s" \
# % (tc.iterators.command, tc.iterators.use_token, sub_command))
if 'penctl' == tc.iterators.command:
if tc.iterators.use_token:
cmd = " -a " + common.PENCTL_TOKEN[n] + \
" %s" % (tc.iterators.subcmd)
else:
cmd = " %s" % (tc.iterators.subcmd)
common.AddPenctlCommand(req, n, cmd)
elif 'ssh' == tc.iterators.command:
cmd = ("ssh -o UserKnownHostsFile=/dev/null "
"-o StrictHostKeyChecking=no root@{} ls -a /".format(
common.GetNaplesMgmtIP(n)))
api.Trigger_AddHostCommand(req, n, cmd)
else:
api.Logger.debug("Invalid cmd:%s" % (tc.iterators.command))
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Main(step):
nodes = api.GetNaplesHostnames()
assert (len(nodes) != 0)
api.Logger.info("Update mode to eth-dev-scale")
req = api.Trigger_CreateExecuteCommandsRequest()
for n in nodes:
common.AddPenctlCommand(
req, n, "create dsc-profile --name scale --num-lifs 16")
common.AddPenctlCommand(
req, n, "create dsc-profile --name default --num-lifs 1")
common.AddPenctlCommand(
req, n, "update dsc --managed-by host --dsc-profile scale")
api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
common.AddPenctlCommand(req, n, "show dsc")
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
for n in tc.nodes:
penctl.AddPenctlCommand(req, n, "show dsc")
api.Logger.info("Loading RDMA driver on nodes: {0}".format(tc.nodes))
for n in tc.nodes:
if tc.os == host.OS_TYPE_LINUX:
api.Trigger_AddHostCommand(req, n, "modprobe ib_uverbs")
api.Trigger_AddHostCommand(req, n, "modprobe rdma_ucm")
api.Trigger_AddHostCommand(
req, n,
"insmod {path}drivers/rdma/drv/ionic/ionic_rdma.ko {opts}".
format(path=tc.iota_path, opts=tc.insmod_opts))
else:
if tc.insmod_opts:
api.Trigger_AddHostCommand(
req, n, "kenv {opts}".format(opts=tc.insmod_opts))
api.Trigger_AddHostCommand(
req, n,
"kldload {path}sys/modules/ionic_rdma/ionic_rdma.ko".format(
path=tc.iota_path))
# allow device to register before proceeding
api.Trigger_AddHostCommand(req, n, "sleep 2")
repeat = int(getattr(tc.args, 'reload', 0))
if repeat:
api.Logger.info(
"Repeating unload + reload {repeat} times".format(repeat=repeat))
for _ in range(repeat):
for n in tc.nodes:
if tc.os == host.OS_TYPE_LINUX:
api.Trigger_AddHostCommand(req, n, "rmmod ionic_rdma")
api.Trigger_AddHostCommand(
req, n,
"insmod {path}drivers/rdma/drv/ionic/ionic_rdma.ko {opts}".
format(path=tc.iota_path, opts=tc.insmod_opts))
else:
api.Trigger_AddHostCommand(req, n, "kldunload ionic_rdma")
api.Trigger_AddHostCommand(
req, n,
"kldload {path}sys/modules/ionic_rdma/ionic_rdma.ko".
format(path=tc.iota_path))
# allow device to register before proceeding
api.Trigger_AddHostCommand(req, n, "sleep 2")
for n in tc.nodes:
if tc.os == host.OS_TYPE_LINUX:
api.Trigger_AddHostCommand(req, n, "lsmod")
else:
api.Trigger_AddHostCommand(req, n, "kldstat")
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
common.AddPenctlCommand(req, n, "%s --help" % (tc.iterators.option))
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Delete(self, req):
cmd = "delete interface --name %s " % self.ltep.interface
cmd += "--encap %s " % self.encap
api.Logger.info("Deleting Tunnel on interface %s" %
(self.ltep.interface))
api.Logger.info("- command: %s" % cmd)
common.AddPenctlCommand(req, self.ltep.node_name, cmd)
self.__add_static_arp(req)
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
common.AddPenctlCommand(req, n, "system tech-support --skip-core")
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def VerifySessionEvents(tc):
cmd_cookie = "show events"
fields = ["APPROACH", "REACHED"]
curr_event_count = GetPenctlEventsCount(tc)
new_event_count = (curr_event_count - tc.event_count_at_start)
api.Logger.info(" Event count at end : %s, New event count: %s" %
(curr_event_count, new_event_count))
req = api.Trigger_CreateExecuteCommandsRequest()
# show events gives out one blank line, we are interested in latest events
# hence use atleast (2+1) for tail command
if new_event_count < 3:
new_event_count = 3
cmd = "show events --json | tail -%s | grep %s" % (new_event_count,
tc.grep_str)
common.AddPenctlCommand(req, tc.wc_server.node_name, cmd)
resp = api.Trigger(req)
cmd_resp = resp.commands[0]
api.PrintCommandResults(cmd_resp)
if cmd_resp.exit_code != 0:
return api.types.status.FAILURE
empty_lines = cmd_resp.stdout.split("\n")
lines = [line for line in empty_lines if line.strip() != ""]
if len(lines) != len(fields):
api.Logger.error("Incorrect events len encountered : %d expected %d" %
(len(lines), len(fields)))
# dump last 10 events for debug purpose
cmd = "show events --json | tail -10 "
common.AddPenctlCommand(req, tc.wc_server.node_name, cmd)
resp = api.Trigger(req)
cmd_resp = resp.commands[0]
api.PrintCommandResults(cmd_resp)
return api.types.status.FAILURE
for i in range(len(lines)):
if fields[i] not in lines[i]:
api.Logger.error(
"Incorrect event encountered :%s expected substring : %s" %
(lines[i], fields[i]))
return api.types.status.FAILURE
api.Logger.info("Event verification session limit for %s Success" %
tc.proto)
return api.types.status.SUCCESS
def Trigger(tc):
tc.cmd_cookies = []
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
common.AddPenctlCommand(req, n, "show events | tail -n 20")
tc.cmd_cookies.append("No events found")
#TODO: Trigger event
#common.AddPenctlCommand(req, n, "show events | tail -n 20")
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def Trigger(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.nodes:
if api.IsNaplesNode(n):
common.AddPenctlCommand(
req, n, "show metrics lif > metrics_lif.out.before")
tc.lif_metrics_old = api.Trigger(req)
common.SendTraffic(tc)
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.nodes:
if api.IsNaplesNode(n):
common.AddPenctlCommand(
req, n, "show metrics lif > metrics.lif.out.after")
tc.lif_metrics_new = api.Trigger(req)
return api.types.status.SUCCESS
def CollectTechSupport(tsName):
try:
#global __CURREN_TECHSUPPORT_CNT
#__CURREN_TECHSUPPORT_CNT = __CURREN_TECHSUPPORT_CNT + 1
if GlobalOptions.pipeline in ["apulu"]:
return types.status.SUCCESS
Logger.info("Collecting techsupport for testsuite {0}".format(tsName))
tsName = re.sub('\W', '_', tsName)
logDir = GlobalOptions.logdir
if not logDir.endswith('/'):
logDir += '/'
logDir += 'techsupport/'
if not os.path.exists(logDir):
os.mkdir(logDir)
nodes = api.GetNaplesHostnames()
req = api.Trigger_CreateExecuteCommandsRequest()
for n in nodes:
Logger.info("Techsupport for node: %s" % n)
common.AddPenctlCommand(
req, n, "system tech-support -b %s-tech-support" % (n))
resp = api.Trigger(req)
result = types.status.SUCCESS
for n, cmd in zip(nodes, resp.commands):
#api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
Logger.error(
"Failed to execute penctl system tech-support on node: %s. err: %d"
% (n, cmd.exit_code))
result = types.status.FAILURE
continue
# Copy tech support tar out
# TAR files are created at: pensando/iota/entities/node1_host/<test_case>
ntsn = "%s-tech-support.tar.gz" % (n)
resp = api.CopyFromHost(n, [ntsn], logDir)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
Logger.error(
"Failed to copy techsupport file %s from node: %s" %
(ntsn, n))
result = types.status.FAILURE
continue
os.rename(logDir + ntsn, logDir + tsName + '_' + ntsn)
#if __CURREN_TECHSUPPORT_CNT > __MAX_TECHSUPPORT_PER_RUN:
# return types.status.CRITICAL
return result
except AttributeError:
Logger.debug('failed to collect tech support. node list not setup yet')
except:
Logger.debug('failed to collect tech support. error was: {0}'.format(
traceback.format_exc()))
return types.status.CRITICAL
def Run():
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
for n in api.GetNaplesHostnames():
enable_sshd = " -a " + common.PENCTL_TOKEN[n] + " system enable-sshd"
#enable_sshd = "system enable-sshd"
common.AddPenctlCommand(req, n, enable_sshd)
copy_key = " -a " + common.PENCTL_TOKEN[
n] + " update ssh-pub-key -f ~/.ssh/id_rsa.pub"
#copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub"
common.AddPenctlCommand(req, n, copy_key)
resp = api.Trigger(req)
for cmd in resp.commands:
if cmd.exit_code != 0:
api.PrintCommandResults(cmd)
return api.types.status.FAILURE
api.Logger.info("DSC ssh is enabled")
for n in api.GetNaplesHostnames():
uuid = GetNaplesUUID(n)
api.SetNaplesNodeUuid(n, uuid)
return api.types.status.SUCCESS
def GetPenctlEventsCount(tc):
req = api.Trigger_CreateExecuteCommandsRequest()
cmd = "show events --json | wc -l"
common.AddPenctlCommand(req, tc.wc_server.node_name, cmd)
resp = api.Trigger(req)
cmd_resp = resp.commands[0]
api.PrintCommandResults(cmd_resp)
if cmd_resp.exit_code != 0:
return api.types.status.FAILURE
curr_event_count = cmd_resp.stdout.strip("\n")
api.Logger.info("Event count : %s " % curr_event_count)
try:
curr_event_count = int(curr_event_count)
except Exception as e:
api.Logger.info(" Failed in retrieving the active event count")
curr_event_count = 0
return curr_event_count
def Main(args):
if GlobalOptions.skip_setup:
# No profile change is required for skip setup
return api.types.status.SUCCESS
newProfile = getattr(args, 'profile', 'FEATURE_PROFILE_BASE')
if newProfile != 'FEATURE_PROFILE_BASE':
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
for n in api.GetNaplesHostnames():
common.AddPenctlCommand(
req, n, "update dsc --device-profile %s" % newProfile)
resp = api.Trigger(req)
if not api.Trigger_IsSuccess(resp):
return api.types.status.FAILURE
api.Logger.info("Naples profile changed to %s - Restarting nodes" %
newProfile)
return api.RestartNodes(api.GetNaplesHostnames())
else:
api.Logger.info("Profile unchanged, input: %s" % newProfile)
return api.types.status.SUCCESS
def Main(tc):
if tc.GetStatus() == api.types.status.SUCCESS:
return api.types.status.SUCCESS
global __CURREN_TECHSUPPORT_CNT
__CURREN_TECHSUPPORT_CNT = __CURREN_TECHSUPPORT_CNT + 1
api.Logger.info("TC: %s failed. Collecting techsupport." % (tc.Name()))
nodes = api.GetNaplesHostnames()
req = api.Trigger_CreateExecuteCommandsRequest()
tc_dir = tc.GetLogsDir()
for n in nodes:
api.Logger.info("Techsupport for node: %s" % n)
common.AddPenctlCommand(req, n,
"system tech-support -b %s-tech-support" % (n))
tc.resp = api.Trigger(req)
result = api.types.status.SUCCESS
for n, cmd in zip(nodes, tc.resp.commands):
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error(
"Failed to execute penctl system tech-support on node: %s. err: %d"
% (n, cmd.exit_code))
result = api.types.status.FAILURE
continue
# Copy tech support tar out
# TAR files are created at: pensando/iota/entities/node1_host/<test_case>
resp = api.CopyFromHost(n, ["%s-tech-support.tar.gz" % (n)], tc_dir)
if resp == None or resp.api_response.api_status != types_pb2.API_STATUS_OK:
api.Logger.error("Failed to copy techsupport file from node: %s" %
n)
result = api.types.status.FAILURE
continue
if __CURREN_TECHSUPPORT_CNT > __MAX_TECHSUPPORT_PER_RUN:
return api.types.status.CRITICAL
return result
def Trigger(tc):
try:
# Get firmware version using penctl
req1 = api.Trigger_CreateExecuteCommandsRequest()
# Get VPD information
req2 = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.naples_nodes:
common.AddPenctlCommand(req1, n.Name(), "show firmware-version")
api.Trigger_AddHostCommand(req2, n.Name(), "lspci -d 1dd8: -vv")
tc.resp1 = api.Trigger(req1)
tc.resp2 = api.Trigger(req2)
# Create a Redfish client object
tc.RF = get_redfish_obj(tc.cimc_info)
tc.setting_details = get_nw_device_functions_settings_info(tc.RF)
except:
api.Logger.error(traceback.format_exc())
return api.types.status.ERROR
return api.types.status.SUCCESS
def Main(step):
if GlobalOptions.skip_setup:
#No mode switch required for skeip setup
return api.types.status.SUCCESS
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
uuidMap = api.GetNaplesNodeUuidMap()
nodes = api.GetNaplesHostnames()
for n in nodes:
# Touch a file to indicate to NMD that the current mode is emulation
cmd = "touch /data/iota-emulation"
api.Trigger_AddNaplesCommand(req, n, cmd)
# Make sure console is enabled
CreateConfigConsoleNoAuth()
api.CopyToNaples(n, [NAPLES_CONFIG_SPEC_LOCAL], "")
cmd = "mv /system-config.json /sysconfig/config0/system-config.json"
api.Trigger_AddNaplesCommand(req, n, cmd)
if common.PenctlGetModeStatus(n) != "NETWORK" or common.PenctlGetTransitionPhaseStatus(n) != "VENICE_REGISTRATION_DONE":
api.Logger.info("Host [{}] is in HOST mode. Initiating mode change.".format(n))
ret = common.SetNaplesModeOOB_Static(n, "1.1.1.1", "1.1.1.2/24")
if ret == None:
return api.types.status.FAILURE
#hack for now, need to set date
cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
num_retries = 60
reboot_nodes = []
while nodes:
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
for n in nodes:
#hack for now, need to set date
api.Logger.info("Checking Transition phase for node : %s" % n)
check_state_cmd = "show naples --json"
time.sleep(30)
common.AddPenctlCommand(req, n, check_state_cmd)
api.Trigger_AddNaplesCommand(req, n, "touch /data/no_watchdog")
resp = api.Trigger(req)
cmd = resp.commands[0]
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Penctl output not in Json format {}".format(cmd.stdout))
return api.types.status.FAILURE
if not hal_show_utils.IsNaplesForwardingModeClassic(n):
api.Logger.info("Dataplane already in HOSTPIN mode. Skipping node [{}] for reboot.".format(n))
reboot_nodes.append(n)
nodes.remove(n)
elif out["status"]["transition-phase"] == "VENICE_UNREACHABLE":
api.Logger.info("Reboot pending on node : %s" % n)
reboot_nodes.append(n)
nodes.remove(n)
elif out["status"]["transition-phase"] == "VENICE_REGISTRATION_DONE":
api.Logger.info("Node already transitioned : %s" % n)
nodes.remove(n)
else:
api.Logger.info("Reboot not pending on node : %s" % n)
time.sleep(1)
num_retries = num_retries - 1
if num_retries == 0:
api.Logger.error("Reboot pending state not transitioned complete on naples")
return api.types.status.FAILURE
ret = api.RestartNodes(reboot_nodes)
if ret != api.types.status.SUCCESS:
api.Logger.error("Node restart failed")
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest(serial = True)
#enable_sshd = "system enable-sshd"
#copy_key = "update ssh-pub-key -f ~/.ssh/id_rsa.pub"
for n in api.GetNaplesHostnames():
#hack for now, need to set date
cmd = "date -s '{}'".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
#common.AddPenctlCommand(req, n, enable_sshd)
#common.AddPenctlCommand(req, n, copy_key)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
result = api.types.status.FAILURE
#Give some time for naples to admit to venice after mode switch
#This is required for now as Iota is setting time manually to make TLS happy
time.sleep(30)
#Check whether naples has switch mode succesfully
req = api.Trigger_CreateExecuteCommandsRequest(serial = False)
for n in nodes:
cmd = "cat /sysconfig/config0/app-start.conf | grep hostpin"
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Mode switch failed on node : {}".format(cmd.node_name))
result = api.types.status.FAILURE
#Verify Agent in right mode.
req = api.Trigger_CreateExecuteCommandsRequest(serial = False)
for n in nodes:
cmd = "curl localhost:8888/api/system/info/"
api.Trigger_AddNaplesCommand(req, n, cmd)
# Delete the iota-emulation file created earlied for NMD. Mode change would have passed by now if it had to.
cmd = "rm -f /data/iota-emulation"
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Agent system get failed : {}".format(cmd.node_name))
result = api.types.status.FAILURE
out = None
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Agent System get out failed {}".format(cmd.stdout))
return api.types.status.FAILURE
if out["naples-mode"] != "NETWORK_MANAGED_OOB":
api.Logger.error("Agent not in correct mode: {} {} ".format(cmd.node_name, out["naples-mode"]))
return api.types.status.FAILURE
api.Logger.info("Trying to re-enable ssh on naples nodes")
return enable_ssh.Main(None)
def Trigger(tc):
#Run all commands in serial
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
tcp_dump_req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
tc.cmd_cookies = []
proto = "tcp"
count = 0
for tunnel in tc.tunnels:
w1 = tunnel.ltep
w2 = tunnel.rtep
w1_class_e_addr = ""
w2_class_e_addr = ""
for sec_ipaddr in w1.sec_ip_addresses:
if ipaddress.IPv4Address(sec_ipaddr).is_reserved:
w1_class_e_addr = sec_ipaddr
api.Logger.info("Class-E IP = %s" % sec_ipaddr)
break
for sec_ipaddr in w2.sec_ip_addresses:
if ipaddress.IPv4Address(sec_ipaddr).is_reserved:
w2_class_e_addr = sec_ipaddr
api.Logger.info("Class-E IP = %s" % sec_ipaddr)
break
if w1_class_e_addr == "" or w2_class_e_addr == "":
api.Logger.info("No Class-E Address found")
return api.types.status.FAILURE
api.Trigger_AddCommand(
req, w1.node_name, w1.workload_name,
"arp -s %s %s" % (w2_class_e_addr, w2.mac_address))
api.Logger.info("Add arp entry for %s" % w2_class_e_addr)
tc.cmd_cookies.append("Add arp entry for %s" % w2_class_e_addr)
api.Trigger_AddCommand(
req, w2.node_name, w2.workload_name,
"arp -s %s %s" % (w1_class_e_addr, w1.mac_address))
api.Logger.info("Add arp entry for %s" % w1_class_e_addr)
tc.cmd_cookies.append("Add arp entry for %s" % w1_class_e_addr)
if count % 2 == 0:
# Traffic to Verify packet is encapped correctly
cmd_cookie = "%s(%s) --> %s(%s)" % (
w1.workload_name, w1.ip_address, w2.workload_name,
w2.ip_address)
nping_cmd = utils.GetNpingCmd(proto, w2.ip_address, 2000,
w1.ip_address, 1)
api.Trigger_AddCommand(req, w1.node_name, w1.workload_name,
nping_cmd)
api.Logger.info("Send " + proto + " traffic from %s" % cmd_cookie)
tc.cmd_cookies.append(proto + " positive test")
# Traffic to Verify packet is encapped and decapped correctly
cmd_cookie1 = "%s(%s) --> %s(%s) IP Opts Dest IP: %s" % (
w1.workload_name, w1_class_e_addr, w2.workload_name,
w2.ip_address, w1.ip_address)
nping_cmd = utils.GetNpingCmd(proto, w2.ip_address, 2000,
w1_class_e_addr)
nping_cmd += " " + GetIPOptionsStr(w1_class_e_addr, w1.ip_address)
api.Trigger_AddCommand(req, w1.node_name, w1.workload_name,
nping_cmd)
api.Logger.info("Send " + proto +
" traffic with valid IP options from %s" %
cmd_cookie1)
tc.cmd_cookies.append(proto + " encap decap test")
# Verify error counter doesn't go up on w1 and w2
penctl_cmd = r"execute halctl show system statistics drop | grep 'Drop Vf IP'"
common.AddPenctlCommand(req, w1.node_name, penctl_cmd)
tc.cmd_cookies.append("Encap decap test statistics")
cmd_cookie2 = "%s(%s) --> %s(%s) IP Opts Dest IP: 14.15.16.17" % (
w1.workload_name, w1.ip_address, w2.workload_name,
w2_class_e_addr)
# Class-E test with invalid IP options
nping_cmd = utils.GetNpingCmd(proto, w2_class_e_addr, 2000,
w1.ip_address)
nping_cmd += " " + GetIPOptionsStr(w1_class_e_addr, "14.15.16.17")
api.Trigger_AddCommand(req, w1.node_name, w1.workload_name,
nping_cmd)
api.Logger.info("Send " + proto +
" traffic with invalid IP options from %s" %
cmd_cookie2)
tc.cmd_cookies.append(proto +
" encap test with invalid IP options")
# Verify error counter goes up on w1
penctl_cmd = r"execute halctl show system statistics drop | grep 'Drop Vf Bad'"
common.AddPenctlCommand(req, w1.node_name, penctl_cmd)
tc.cmd_cookies.append(
"Encap test with invalid IP options statistics")
if count % 2 == 1:
cmd_cookie3 = "%s(%s) --> %s(%s) IP Opts Dest IP: 0.0.0.0" % (
w1.workload_name, w1.ip_address, w2.workload_name,
w2_class_e_addr)
# Class-E test with zero as dest IP in IP options
nping_cmd = utils.GetNpingCmd(proto, w2_class_e_addr, 2000,
w1.ip_address)
nping_cmd += " " + GetIPOptionsStr(w1_class_e_addr, "0.0.0.0")
api.Trigger_AddCommand(req, w1.node_name, w1.workload_name,
nping_cmd)
api.Logger.info(
"Send " + proto +
" traffic with zero as dest IP in IP options from %s" %
cmd_cookie3)
tc.cmd_cookies.append(proto +
" encap test with 0.0.0.0 IP options")
# Verify error counter goes up on w1
penctl_cmd = r"execute halctl show system statistics drop | grep 'Drop Vf Bad'"
common.AddPenctlCommand(req, w1.node_name, penctl_cmd)
tc.cmd_cookies.append(
"Encap test with 0.0.0.0 IP options statistics")
cmd_cookie4 = "%s(%s) --> %s(%s) IP Opts Dest IP: 224.0.0.1" % (
w1.workload_name, w1.ip_address, w2.workload_name,
w2_class_e_addr)
# Class-E test with multicast dest IP in IP options
nping_cmd = utils.GetNpingCmd(proto, w2_class_e_addr, 2000,
w1.ip_address)
nping_cmd += " " + GetIPOptionsStr(w1_class_e_addr, "224.0.0.1")
api.Trigger_AddCommand(req, w1.node_name, w1.workload_name,
nping_cmd)
api.Logger.info(
"Send " + proto +
" traffic with multicast dest IP in IP options from %s" %
cmd_cookie4)
tc.cmd_cookies.append(proto +
" encap test with multicast IP options")
# Verify error counter goes up on w1
penctl_cmd = r"execute halctl show system statistics drop | grep 'Drop Vf Bad'"
common.AddPenctlCommand(req, w1.node_name, penctl_cmd)
tc.cmd_cookies.append(
"Encap test with multicast IP options statistics")
count += 1
if proto is "tcp":
proto = "udp"
else:
proto = "tcp"
if count == 16:
break
tc.resp = api.Trigger(req)
return api.types.status.SUCCESS
def checkUpdateMode():
req = api.Trigger_CreateExecuteCommandsRequest()
for n in tc.Nodes:
# TODO Add more negative test cases
common.AddPenctlCommand(req, n,
"create dsc-profile -n default -i 42")
common.AddPenctlCommand(req, n, "delete dsc-profile -n default")
common.AddPenctlCommand(
req, n, "delete dsc-profile -n nonExistentProfile")
common.AddPenctlCommand(req, n,
"update dsc -o host -f scale -c a:a")
common.AddPenctlCommand(req, n,
"update dsc -o network -k oob -c a:a")
common.AddPenctlCommand(
req, n, "update dsc -o network -k inband -c a -c a1 -c a2:1")
common.AddPenctlCommand(req, n, "update dsc -o a")
common.AddPenctlCommand(req, n, "update dsc -k a")
common.AddPenctlCommand(req, n, "update dsc -o network -k a")
common.AddPenctlCommand(req, n,
"update dsc -o network -k oob -m a")
common.AddPenctlCommand(req, n,
"update dsc -o network -n a: -k inband")
common.AddPenctlCommand(req, n, "update dsc -n a: -f default")
common.AddPenctlCommand(req, n,
"update dsc -o network -k oob -p a")
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 1:
return api.types.status.FAILURE
for n, cmd in zip(tc.Nodes, resp.commands):
cmd_resp = list(filter(None, cmd.stdout.split("\n")))
if not (x in cmd_resp for x in def_mode_set_error_strs):
api.Logger.error(
"Update mode response not valid : %s" %
(set(def_mode_set_error_strs) - set(cmd_resp)))
return api.types.status.FAILURE
return api.types.status.SUCCESS
def Main(step):
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
venice_ips = api.GetVeniceMgmtIpAddresses()
assert (venice_ips)
uuidMap = api.GetNaplesNodeUuidMap()
for n in api.GetNaplesHostnames():
#hack for now, need to set date
cmd = "date -s '{}'".format(
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
api.Logger.info("Running mode switch on node : %s" % n)
mode_switch_cmd = "update naples --managed-by network --management-network oob --hostname {} --primary-mac {}".format(
__get_mgmt_nw(), n, uuidMap[n])
common.AddPenctlCommand(req, n, mode_switch_cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
nodes = api.GetNaplesHostnames()
num_retries = 60
while nodes:
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
for n in nodes:
#hack for now, need to set date
api.Logger.info("Checking Tranisition phase for node : %s" % n)
check_state_cmd = "show naples --json"
common.AddPenctlCommand(req, n, check_state_cmd)
resp = api.Trigger(req)
cmd = resp.commands[0]
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
return api.types.status.FAILURE
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Penctl output not in Json format {}".format(
cmd.stdout))
return api.types.status.FAILURE
if out["status"]["transition-phase"] == "REBOOT_PENDING":
api.Logger.info("Reboot pending on node : %s" % n)
nodes.remove(n)
else:
api.Logger.info("Reboot not pending on node : %s" % n)
time.sleep(1)
num_retries = num_retries - 1
if num_retries == 0:
api.Logger.error(
"Reboot pending state not transitioned complete on naples")
return api.types.status.FAILURE
reboot_nodes = []
for n in api.GetNaplesHostnames():
reboot_nodes.append(n)
ret = api.RestartNodes(reboot_nodes)
if ret != api.types.status.SUCCESS:
api.Logger.error("Node restart failed")
return api.types.status.FAILURE
req = api.Trigger_CreateExecuteCommandsRequest(serial=True)
for n in api.GetNaplesHostnames():
#hack for now, need to set date
cmd = "date -s '{}'".format(
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
result = api.types.status.FAILURE
#Give some time for naples to admit to venice after mode switch
#This is required for now as Iota is setting time manually to make TLS happy
time.sleep(30)
#Check whether naples has switch mode succesfully
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for n in api.GetNaplesHostnames():
cmd = "cat /sysconfig/config0/app-start.conf | grep hostpin"
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Mode switch failed on node : {}".format(
cmd.node_name))
result = api.types.status.FAILURE
#Verify Agent in right mode.
req = api.Trigger_CreateExecuteCommandsRequest(serial=False)
for n in api.GetNaplesHostnames():
cmd = "curl localhost:8888/api/system/info/"
api.Trigger_AddNaplesCommand(req, n, cmd)
resp = api.Trigger(req)
venice_urls = []
for ip in api.GetVeniceMgmtIpAddresses():
venice_urls.append(ip + ":9009")
for cmd in resp.commands:
api.PrintCommandResults(cmd)
if cmd.exit_code != 0:
api.Logger.error("Agent system get failed : {}".format(
cmd.node_name))
result = api.types.status.FAILURE
out = None
try:
out = json.loads(cmd.stdout)
except:
api.Logger.error("Agent System get out failed {}".format(
cmd.stdout))
return api.types.status.FAILURE
if out["naples-mode"] != "NETWORK_MANAGED_OOB":
api.Logger.error("Agent not in correct mode: {} {} ".format(
cmd.node_name, out["naples-mode"]))
return api.types.status.FAILURE
if set(out["controller-ips"]) != set(venice_urls):
api.Logger.error("Agent controller IPs don't match: {} {}".format(
venice_urls, out["controller-ips"]))
return api.types.status.FAILURE
return api.types.status.SUCCESS
