def cmdhost(cmd, cwd=None, logfile=None, silent=False):
retval, result = CMD.shell(cmd, cwd=cwd, logfile=logfile, silent=silent, DEBUG=DEBUG)
if not silent:
LOG.Info("Finished \"%s\"\n" % cmd, silent=silent)
LOG.Info("\n".join(result))
return retval
def RUN_STEP(num_of_step):
if args.start <= num_of_step and args.stop >= num_of_step:
LOG.Info("#######################")
LOG.Time("# Step %d Started." % num_of_step)
dplorch["current_step"] = num_of_step
return True
LOG.Info("#######################")
LOG.Time("# Step %d Skipped." % num_of_step)
return False
def __mount_point(self, mnt_point, iso):
if os.listdir(mnt_point):
LOG.Info('{} is already mounted, umounting'.format(mnt_point))
self.__umount_point(mnt_point)
ret, log = CMD.shell('sudo mount {0} {1}'.format(self.iso, mnt_point))
if ret == 0:
LOG.Info('ISO mounted on {}'.format(mnt_point))
else:
__print_errorlog("MOUNT", log)
exit(1)
def FINISH_STEP(num_of_step):
LOG.Time("Step %d Finished" % num_of_step)
if args.stop <= num_of_step:
LOG.Info("#################################")
LOG.Info("Test Finished!!!")
dplmnt['status'] = "Finished"
dplmnt['endtime'] = datetime.now().strftime('%H:%M:%S')
LOG.Time("<<<Deployment Finished>>>")
if args.log and num_of_step > STEP_CONTROLLER0:
STX.get_system_logs(node_0)
exit_with_success()
def __wait_for_new_host(node0, node, minutes):
hostname = node.get_hostname()
nodeid = node.get_nodeid()
personality = node.get_personality()
node.power_off()
node.boot()
def check():
retval, retlog = node0.stx_cmd("system host-list | grep %d" % nodeid,
silent=True)
return retval == 0
ret = UTILS.wait_for("New Host %d occur." % nodeid,
check,
slowdown=STX_SLOWDOWN).run(minutes)
if ret == 0:
LOG.Info("Found host node (#%d) as %s, set personality as %s." %
(nodeid, hostname, personality))
CK_RET(
node0.stx_cmd("system host-update %d personality=%s hostname=%s "
"boot_device=%s rootfs_device=%s "
"console=tty0" %
(nodeid, personality, hostname,
node.get_boot_device(), node.get_rootfs_device())))
return True
else:
LOG.Error("Failed to found new host node %s." % hostname)
return False
def check_pxe_services():
"""
"""
services = ['dnsmasq']
for service in services:
LOG.Info("Restarting %s ..." % service)
CMD.shell('sudo systemctl restart {}'.format(service))
def stx_ansible(node, localhost_config):
LOG.Info("# Config Controller with config file %s" % localhost_config)
CK_RET(node.copy_to_node(localhost_config, "~/"))
# Using ansible
cmd = []
cmd.append("export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin")
cmd.append("export HOSTNAME=localhost")
cmd.append(
"BOOTSTRAP_YML=`find /usr/share/ansible/stx-ansible/playbooks -name bootstrap.yml`"
)
cmd.append("ansible-playbook "
"$BOOTSTRAP_YML "
"-e \"ansible_become_pass=%s\"" % node.get_password())
retval, retlog = node.ssh(
";".join(cmd), logfile=getlogfile("controller-0.00_ansible.log"))
LOG.print_log(retlog)
if retval == 0:
for l in retlog:
if l.find("failed=") >= 0:
import re
failed = re.compile(".*failed=(\d*)").match(l).group(1)
if failed != "0":
LOG.Error(
"##### ERROR: ansible doesn't return good result.")
return -1
else:
LOG.Error("##### ERROR: ansible script exited abnormally.")
return -1
return 0
def secure_ssh(cmd,
server,
user,
password=None,
cwd=None,
logfile=None,
silent=False):
if not silent:
LOG.Info("EXEC >>> %s@%s: \"%s\"" % (user, server, cmd))
head = "ssh -t -oStrictHostKeyChecking=no -oCheckHostIP=no " \
"%s@%s" % (user, server)
if password:
sshcmd = "%s %s" % (head, cmd)
retval, retlog = run_expect_cmd_with_password(sshcmd,
password,
cwd=cwd,
logfile=logfile,
silent=silent)
for l in retlog:
if l.find("spawn id exp4 not open") >= 0:
retval = 1
break
else:
sshcmd = "%s 'echo \"%s\"; %s' 2>&1" % (head, SSH_LOG_FLAG, cmd)
retval, retlog = CMD.shell(sshcmd,
cwd=cwd,
logfile=logfile,
silent=silent)
remove_ssh_log_head(retlog)
return retval, retlog
def close_pxe_services():
"""
"""
services = ['dnsmasq']
for service in services:
LOG.Info("Stopping %s ..." % service)
CMD.shell('sudo systemctl stop {}'.format(service))
def default_route_on_nic(nicname):
ret, log = self.ssh('sudo route | grep default | grep %s' %
nicname,
sudo=True,
silent=True)
if ret == 0:
LOG.Info("Already has default route on %s" % nicname)
return ret == 0
def get_pod_logs(node, logname=None):
node.ssh("./get_pods_log.sh", silent=True)
logloc = getlogfile(logname)
if not logloc:
logloc = getlogfile("pod_logs")
node.copy_from_node("~/pod_logs", logloc)
LOG.Info("Please check the pod logs under %s." % logloc)
def exit_if_true():
stat, progress = __stx_app_stat(node0, appname)
LOG.Info("current status: %s, %s" % (stat, progress))
if stat == None or stat == current_stat:
# app not found or app status no change, keep waiting
return False
else:
# status changed, exit the wait loop
return True
def main():
import argparse
parser = argparse.ArgumentParser(description="Node Controller")
parser.add_argument('vmname', help="VM name", type=str)
parser.add_argument('node_config', help="Test Node Config.", type=str)
args = parser.parse_args()
node = Node_KVM(args.node_config, args.vmname, HOSTNAME_CONTROLLER0, 1,
"10.10.10.3")
ret = node.create_secure_path()
LOG.Info("== Create Secure Path: ret %s" % ret)
ret, log = node.ssh("ls")
LOG.Info("== SSH CMD: ret %s" % ret)
LOG.print_log(log)
print(node.get_oam_ip())
print(node.get_nic(name="eth1001"))
print(node.get_nic(bridge="virbr3"))
print(node.is_power_on())
def add_route_for_oam(self, oam_gate_way):
oam_ip = self.get_oam_ip()
oam_bridge = self.get_oam_bridge()
if oam_ip and oam_bridge:
LOG.Info("Add route for oam %s via %s on dev %s" %
(oam_ip, oam_gate_way, oam_bridge))
ret, log = CMD.shell("sudo ip route add %s via %s dev %s" %
(oam_ip, oam_gate_way, oam_bridge))
if ret:
LOG.print_error(log)
def cleanup_network_env(self, oam_gate_way):
default_nic = None
for n in self.CONFIG['network'].keys():
if n == 'default':
default_nic = self.CONFIG['network'][n]
continue
nic = self.CONFIG['network'][n]
if 'brname' in nic and nic['brname']:
ret, log = CMD.shell("sudo ip addr | grep %s | grep %s" %
(oam_gate_way, nic['brname']),
silent=True)
if ret == 0:
LOG.Info("Delete %s on %s." %
(oam_gate_way, nic['brname']))
ret, log = CMD.shell("sudo ip addr del %s dev %s" %
(oam_gate_way, nic['brname']))
def default_route_on_nic(nicname):
ret, log = self.ssh('sudo route | grep default | grep %s' %
nicname,
sudo=True,
silent=True)
if ret == 0:
LOG.Info("Already has default route on %s" % nicname)
return ret == 0
for n in self.CONFIG['network'].keys():
if n == 'default':
continue
nic = self.CONFIG['network'][n]
if default_route_on_nic(nic['name']):
LOG.Info("Delete default route on %s." % nic['name'])
self.ssh('sudo route del default', sudo=True)
if default_nic and not default_route_on_nic(default_nic['name']):
LOG.Info("Add default route %s on %s" %
(oam_gate_way, default_nic['name']))
self.ssh('sudo route add default gw %s %s' %
(oam_gate_way, default_nic['name']),
sudo=True)
def get_system_logs(node):
syslog_folder = getlogfile("system_logs")
CMD.shell("mkdir -p %s/" % syslog_folder, silent=True)
ret, retlog = node.ssh("collect -a", forcesudo=True, silent=True)
if ret != 0:
LOG.Warning("collect returned error code: %s" % str(ret))
LOG.print_warning(retlog)
node.copy_from_node("/scratch/*.tgz", syslog_folder)
node.ssh('sudo sm-dump',
logfile=getlogfile("sm-dump.log"),
sudo=True,
silent=True)
LOG.Info("Please check the full system log under %s." % syslog_folder)
def create_secure_path(server, user, password):
cmd = "ssh -t -oStrictHostKeyChecking=no -oCheckHostIP=no " \
"%s@%s mkdir -p ~/.ssh" % (user, server)
retval, retlog = run_expect_cmd_with_password(cmd, password)
if retval != 0:
return retval, retlog
if not os.path.exists(TEST_PUBKEY):
LOG.Info("pubkey %s not existing, generate new key" % TEST_PUBKEY)
ret, log = CMD.shell("ssh-keygen -o -f %s -P \"\"" % TEST_KEY)
LOG.print_log(log)
add_sshkey_config(server, user)
retval, retlog = scp_to_server(TEST_PUBKEY, "~/.ssh/authorized_keys",
server, user, password)
return retval, retlog
def __umount_point(self, mnt_point):
umounting_attempts = 3
while umounting_attempts > 0:
ret, log = CMD.shell('sudo umount -l {}'.format(mnt_point))
if ret != 0 and umounting_attempts:
LOG.Info('Failed to umount {}, retrying...'.format(
mnt_point))
elif ret != 0 and not umounting_attempts:
LOG.Error('Max umounting attempts reached, leaving '
'installation')
__print_errorlog("UMOUNT", log)
exit(1)
else:
break
umounting_attempts -= 1
choices=["kvm"], # removed "vbox" because vbox is not available now.
default="kvm")
args = parser.parse_args()
def iskvm():
return args.method == "kvm"
def isvbox():
return args.method == "vbox"
if isvbox():
LOG.Info("!!! WARNING !!!: vbox is not tested in this version of script.")
def cmdhost(cmd, cwd=None, logfile=None, silent=False):
realcmd = cmd
result = ""
retval = 0
if cwd != None:
os.chdir(cwd)
try:
result = check_output(realcmd, shell=True).splitlines()
except CalledProcessError as ecp:
LOG.Error("ERROR: failed to run \"%s\": returned %s %s" %
(cmd, ecp.returncode, ecp.output))
retval = ecp.returncode
return config_file
def copy_provision_script_to_node(node):
if iskvm() and isinstance(node, NODE.Node_KVM):
CK_RET(node.kvm_update_netmap())
CK_RET(node.kvm_update_nicname())
script_folder = os.path.join(dplmnt["testdir"], 'needcopy')
if not os.path.exists(script_folder):
UTILS.copy_folder(os.path.join(CurrPath, "stx_provision"), script_folder)
CK_RET(node.copy_to_node(script_folder + "/*", "~/"))
CK_RET(node.ssh("sudo chmod +x *.sh; sudo chmod +x */*.sh", sudo=True))
LOG.Info("##########################")
LOG.Info("# Start to deploy %s" % dplmnt["iso"])
LOG.Info("# helm charts %s" % dplmnt["helm_charts_list"])
LOG.Info("# start from Step %s" % args.start)
LOG.Info("# stop after Step %s" % args.stop)
LOG.Info("# docker proxy %s" % dplmnt["docker_proxy"])
LOG.Time("<<<Deployment Started>>>")
LOG.Info("# Deployment method: %s with %s" % (dplmnt["system_mode"],
dplmnt["method"]))
## Test Nodes
LOG.Info("# Num of Controllers: %d" % dplmnt['controller_num'])
for n in get_controllers():
node = dplorch["nodes"][n]
LOG.Info("# %s @ %s" % (node.get_hostname(), node.get_oam_ip()))
if dplmnt['system_mode'] != "simplex":
LOG.Info("# Floating IP @ %s" % get_floating_ip())
def Info(self, log):
LOG.Info("%s: %s" % (self.name, log))
vmlist = []
vmlist_d = []
if args.vmname:
cmd = "sudo virsh list --all | grep %s | awk '{print $2}'" % args.vmname
retval, vmlist = CMD.shell(cmd)
else:
cmd = "sudo virsh list --all | grep running | awk '{print $2}'"
retval, vmlist = CMD.shell(cmd)
cmd = "sudo virsh list --all | grep -v running | grep -v Name | awk '{print $2}'"
retval, vmlist_d = CMD.shell(cmd)
while '' in vmlist_d:
vmlist_d.remove('')
for r in vmlist:
if is_on_the_bridge(r, args.brname):
LOG.Info("===========================")
LOG.Info("Destroying : %s" % r)
cmds = []
# Only destroy the vms here
cmds.append("sudo virsh destroy %s || true" % r)
rv, rs = CMD.shell(";".join(cmds))
LOG.Info("Return Value: %s" % str(rv))
for rl in rs:
LOG.Info(rl)
if args.delete_all:
LOG.Info("!!!!!!!!!!!!!!!!!!!!!!!!!!!")
LOG.Info("Vms will be perminantly deleted.")
for r in vmlist + vmlist_d:
if is_on_the_bridge(r, args.brname):
LOG.Info("===========================")
def __init__(self, msg, check_func, interval=10, slowdown=1):
LOG.Info("Waiting for " + msg)
self.check_func = check_func
self.interval = interval
self.slowdown = slowdown
def stx_apply_application(node0, helm_charts):
LOG.Info("Applying application with helm charts %s." % helm_charts)
# copy the helm_charts to test folder and upload it to the target.
charts_filename = os.path.basename(helm_charts)
appname = os.path.splitext(charts_filename)[0]
node0.copy_to_node(helm_charts, "~/")
failures = 0
appstat = __stx_app_stat(node0, appname)[0]
def handle_failure(failures):
failures += 1
get_pod_logs(node0, logname="pod_logs_failed_%d" % failures)
return failures
while appstat != APP_APPLY_SUCCESS:
LOG.Info("App %s current status: %s" % (appname, appstat))
if appstat == APP_NONE:
retval, retlog = __stx_app_op(node0,
APP_OP_UPLOAD,
helm_charts=charts_filename)
real_app_name = None
for l in retlog:
if l.find(" name ") >= 0:
real_app_name = l.split("|")[2].strip()
if not real_app_name:
__exit_with_exception()
LOG.Info("App Real Name: %s" % real_app_name)
appname = real_app_name
appstat = wait_for_app_status_finish(node0, appname, 20,
APP_UPLOADING)
elif appstat == APP_UPLOAD_SUCCESS:
retval, retlog = __stx_app_op(node0, APP_OP_APPLY, appname=appname)
appstat = wait_for_app_status_finish(node0, appname, 120,
APP_APPLYING)
elif appstat == APP_UPLOAD_FAILED:
failures = handle_failure(failures)
retval, retlog = __stx_app_op(node0,
APP_OP_DELETE,
appname=appname)
appstat = wait_for_app_status_finish(node0, appname, 20,
APP_DELETING)
elif appstat == APP_APPLY_SUCCESS:
# should not run here
break
elif appstat == APP_APPLY_FAILED:
failures = handle_failure(failures)
retval, retlog = __stx_app_op(node0,
APP_OP_REMOVE,
appname=appname)
appstat = wait_for_app_status_finish(node0, appname, 30,
APP_REMOVING)
elif appstat == APP_REMOVE_FAILED:
failures = handle_failure(failures)
retval, retlog = __stx_app_op(node0,
APP_OP_REMOVE,
appname=appname)
appstat = wait_for_app_status_finish(node0, appname, 30,
APP_REMOVING)
elif appstat == APP_DELETE_FAILED:
# Fatal
LOG.Error("Failed to delete app %s after %d failures" %
(appname, failures))
__exit_with_exception()
elif appstat in [
APP_UPLOADING, APP_APPLYING, APP_REMOVING, APP_DELETING
]:
failures = handle_failure(failures)
appstat = wait_for_app_status_finish(node0, appname, 20, appstat)
else:
# Fatal
LOG.Error("Unknown application status %s" % appstat)
__exit_with_exception()
if failures >= 5:
# Fatal
LOG.Error("Application %s apply failed!" % app_name)
__exit_with_exception()
return appname