def cmdhost(cmd, cwd=None, logfile=None, silent=False): retval, result = CMD.shell(cmd, cwd=cwd, logfile=logfile, silent=silent, DEBUG=DEBUG) if not silent: LOG.Info("Finished \"%s\"\n" % cmd, silent=silent) LOG.Info("\n".join(result)) return retval
def RUN_STEP(num_of_step): if args.start <= num_of_step and args.stop >= num_of_step: LOG.Info("#######################") LOG.Time("# Step %d Started." % num_of_step) dplorch["current_step"] = num_of_step return True LOG.Info("#######################") LOG.Time("# Step %d Skipped." % num_of_step) return False
def __mount_point(self, mnt_point, iso): if os.listdir(mnt_point): LOG.Info('{} is already mounted, umounting'.format(mnt_point)) self.__umount_point(mnt_point) ret, log = CMD.shell('sudo mount {0} {1}'.format(self.iso, mnt_point)) if ret == 0: LOG.Info('ISO mounted on {}'.format(mnt_point)) else: __print_errorlog("MOUNT", log) exit(1)
def FINISH_STEP(num_of_step): LOG.Time("Step %d Finished" % num_of_step) if args.stop <= num_of_step: LOG.Info("#################################") LOG.Info("Test Finished!!!") dplmnt['status'] = "Finished" dplmnt['endtime'] = datetime.now().strftime('%H:%M:%S') LOG.Time("<<<Deployment Finished>>>") if args.log and num_of_step > STEP_CONTROLLER0: STX.get_system_logs(node_0) exit_with_success()
def __wait_for_new_host(node0, node, minutes): hostname = node.get_hostname() nodeid = node.get_nodeid() personality = node.get_personality() node.power_off() node.boot() def check(): retval, retlog = node0.stx_cmd("system host-list | grep %d" % nodeid, silent=True) return retval == 0 ret = UTILS.wait_for("New Host %d occur." % nodeid, check, slowdown=STX_SLOWDOWN).run(minutes) if ret == 0: LOG.Info("Found host node (#%d) as %s, set personality as %s." % (nodeid, hostname, personality)) CK_RET( node0.stx_cmd("system host-update %d personality=%s hostname=%s " "boot_device=%s rootfs_device=%s " "console=tty0" % (nodeid, personality, hostname, node.get_boot_device(), node.get_rootfs_device()))) return True else: LOG.Error("Failed to found new host node %s." % hostname) return False
def check_pxe_services(): """ """ services = ['dnsmasq'] for service in services: LOG.Info("Restarting %s ..." % service) CMD.shell('sudo systemctl restart {}'.format(service))
def stx_ansible(node, localhost_config): LOG.Info("# Config Controller with config file %s" % localhost_config) CK_RET(node.copy_to_node(localhost_config, "~/")) # Using ansible cmd = [] cmd.append("export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin") cmd.append("export HOSTNAME=localhost") cmd.append( "BOOTSTRAP_YML=`find /usr/share/ansible/stx-ansible/playbooks -name bootstrap.yml`" ) cmd.append("ansible-playbook " "$BOOTSTRAP_YML " "-e \"ansible_become_pass=%s\"" % node.get_password()) retval, retlog = node.ssh( ";".join(cmd), logfile=getlogfile("controller-0.00_ansible.log")) LOG.print_log(retlog) if retval == 0: for l in retlog: if l.find("failed=") >= 0: import re failed = re.compile(".*failed=(\d*)").match(l).group(1) if failed != "0": LOG.Error( "##### ERROR: ansible doesn't return good result.") return -1 else: LOG.Error("##### ERROR: ansible script exited abnormally.") return -1 return 0
def secure_ssh(cmd, server, user, password=None, cwd=None, logfile=None, silent=False): if not silent: LOG.Info("EXEC >>> %s@%s: \"%s\"" % (user, server, cmd)) head = "ssh -t -oStrictHostKeyChecking=no -oCheckHostIP=no " \ "%s@%s" % (user, server) if password: sshcmd = "%s %s" % (head, cmd) retval, retlog = run_expect_cmd_with_password(sshcmd, password, cwd=cwd, logfile=logfile, silent=silent) for l in retlog: if l.find("spawn id exp4 not open") >= 0: retval = 1 break else: sshcmd = "%s 'echo \"%s\"; %s' 2>&1" % (head, SSH_LOG_FLAG, cmd) retval, retlog = CMD.shell(sshcmd, cwd=cwd, logfile=logfile, silent=silent) remove_ssh_log_head(retlog) return retval, retlog
def close_pxe_services(): """ """ services = ['dnsmasq'] for service in services: LOG.Info("Stopping %s ..." % service) CMD.shell('sudo systemctl stop {}'.format(service))
def default_route_on_nic(nicname): ret, log = self.ssh('sudo route | grep default | grep %s' % nicname, sudo=True, silent=True) if ret == 0: LOG.Info("Already has default route on %s" % nicname) return ret == 0
def get_pod_logs(node, logname=None): node.ssh("./get_pods_log.sh", silent=True) logloc = getlogfile(logname) if not logloc: logloc = getlogfile("pod_logs") node.copy_from_node("~/pod_logs", logloc) LOG.Info("Please check the pod logs under %s." % logloc)
def exit_if_true(): stat, progress = __stx_app_stat(node0, appname) LOG.Info("current status: %s, %s" % (stat, progress)) if stat == None or stat == current_stat: # app not found or app status no change, keep waiting return False else: # status changed, exit the wait loop return True
def main(): import argparse parser = argparse.ArgumentParser(description="Node Controller") parser.add_argument('vmname', help="VM name", type=str) parser.add_argument('node_config', help="Test Node Config.", type=str) args = parser.parse_args() node = Node_KVM(args.node_config, args.vmname, HOSTNAME_CONTROLLER0, 1, "10.10.10.3") ret = node.create_secure_path() LOG.Info("== Create Secure Path: ret %s" % ret) ret, log = node.ssh("ls") LOG.Info("== SSH CMD: ret %s" % ret) LOG.print_log(log) print(node.get_oam_ip()) print(node.get_nic(name="eth1001")) print(node.get_nic(bridge="virbr3")) print(node.is_power_on())
def add_route_for_oam(self, oam_gate_way): oam_ip = self.get_oam_ip() oam_bridge = self.get_oam_bridge() if oam_ip and oam_bridge: LOG.Info("Add route for oam %s via %s on dev %s" % (oam_ip, oam_gate_way, oam_bridge)) ret, log = CMD.shell("sudo ip route add %s via %s dev %s" % (oam_ip, oam_gate_way, oam_bridge)) if ret: LOG.print_error(log)
def cleanup_network_env(self, oam_gate_way): default_nic = None for n in self.CONFIG['network'].keys(): if n == 'default': default_nic = self.CONFIG['network'][n] continue nic = self.CONFIG['network'][n] if 'brname' in nic and nic['brname']: ret, log = CMD.shell("sudo ip addr | grep %s | grep %s" % (oam_gate_way, nic['brname']), silent=True) if ret == 0: LOG.Info("Delete %s on %s." % (oam_gate_way, nic['brname'])) ret, log = CMD.shell("sudo ip addr del %s dev %s" % (oam_gate_way, nic['brname'])) def default_route_on_nic(nicname): ret, log = self.ssh('sudo route | grep default | grep %s' % nicname, sudo=True, silent=True) if ret == 0: LOG.Info("Already has default route on %s" % nicname) return ret == 0 for n in self.CONFIG['network'].keys(): if n == 'default': continue nic = self.CONFIG['network'][n] if default_route_on_nic(nic['name']): LOG.Info("Delete default route on %s." % nic['name']) self.ssh('sudo route del default', sudo=True) if default_nic and not default_route_on_nic(default_nic['name']): LOG.Info("Add default route %s on %s" % (oam_gate_way, default_nic['name'])) self.ssh('sudo route add default gw %s %s' % (oam_gate_way, default_nic['name']), sudo=True)
def get_system_logs(node): syslog_folder = getlogfile("system_logs") CMD.shell("mkdir -p %s/" % syslog_folder, silent=True) ret, retlog = node.ssh("collect -a", forcesudo=True, silent=True) if ret != 0: LOG.Warning("collect returned error code: %s" % str(ret)) LOG.print_warning(retlog) node.copy_from_node("/scratch/*.tgz", syslog_folder) node.ssh('sudo sm-dump', logfile=getlogfile("sm-dump.log"), sudo=True, silent=True) LOG.Info("Please check the full system log under %s." % syslog_folder)
def create_secure_path(server, user, password): cmd = "ssh -t -oStrictHostKeyChecking=no -oCheckHostIP=no " \ "%s@%s mkdir -p ~/.ssh" % (user, server) retval, retlog = run_expect_cmd_with_password(cmd, password) if retval != 0: return retval, retlog if not os.path.exists(TEST_PUBKEY): LOG.Info("pubkey %s not existing, generate new key" % TEST_PUBKEY) ret, log = CMD.shell("ssh-keygen -o -f %s -P \"\"" % TEST_KEY) LOG.print_log(log) add_sshkey_config(server, user) retval, retlog = scp_to_server(TEST_PUBKEY, "~/.ssh/authorized_keys", server, user, password) return retval, retlog
def __umount_point(self, mnt_point): umounting_attempts = 3 while umounting_attempts > 0: ret, log = CMD.shell('sudo umount -l {}'.format(mnt_point)) if ret != 0 and umounting_attempts: LOG.Info('Failed to umount {}, retrying...'.format( mnt_point)) elif ret != 0 and not umounting_attempts: LOG.Error('Max umounting attempts reached, leaving ' 'installation') __print_errorlog("UMOUNT", log) exit(1) else: break umounting_attempts -= 1
choices=["kvm"], # removed "vbox" because vbox is not available now. default="kvm") args = parser.parse_args() def iskvm(): return args.method == "kvm" def isvbox(): return args.method == "vbox" if isvbox(): LOG.Info("!!! WARNING !!!: vbox is not tested in this version of script.") def cmdhost(cmd, cwd=None, logfile=None, silent=False): realcmd = cmd result = "" retval = 0 if cwd != None: os.chdir(cwd) try: result = check_output(realcmd, shell=True).splitlines() except CalledProcessError as ecp: LOG.Error("ERROR: failed to run \"%s\": returned %s %s" % (cmd, ecp.returncode, ecp.output)) retval = ecp.returncode
return config_file def copy_provision_script_to_node(node): if iskvm() and isinstance(node, NODE.Node_KVM): CK_RET(node.kvm_update_netmap()) CK_RET(node.kvm_update_nicname()) script_folder = os.path.join(dplmnt["testdir"], 'needcopy') if not os.path.exists(script_folder): UTILS.copy_folder(os.path.join(CurrPath, "stx_provision"), script_folder) CK_RET(node.copy_to_node(script_folder + "/*", "~/")) CK_RET(node.ssh("sudo chmod +x *.sh; sudo chmod +x */*.sh", sudo=True)) LOG.Info("##########################") LOG.Info("# Start to deploy %s" % dplmnt["iso"]) LOG.Info("# helm charts %s" % dplmnt["helm_charts_list"]) LOG.Info("# start from Step %s" % args.start) LOG.Info("# stop after Step %s" % args.stop) LOG.Info("# docker proxy %s" % dplmnt["docker_proxy"]) LOG.Time("<<<Deployment Started>>>") LOG.Info("# Deployment method: %s with %s" % (dplmnt["system_mode"], dplmnt["method"])) ## Test Nodes LOG.Info("# Num of Controllers: %d" % dplmnt['controller_num']) for n in get_controllers(): node = dplorch["nodes"][n] LOG.Info("# %s @ %s" % (node.get_hostname(), node.get_oam_ip())) if dplmnt['system_mode'] != "simplex": LOG.Info("# Floating IP @ %s" % get_floating_ip())
def Info(self, log): LOG.Info("%s: %s" % (self.name, log))
vmlist = [] vmlist_d = [] if args.vmname: cmd = "sudo virsh list --all | grep %s | awk '{print $2}'" % args.vmname retval, vmlist = CMD.shell(cmd) else: cmd = "sudo virsh list --all | grep running | awk '{print $2}'" retval, vmlist = CMD.shell(cmd) cmd = "sudo virsh list --all | grep -v running | grep -v Name | awk '{print $2}'" retval, vmlist_d = CMD.shell(cmd) while '' in vmlist_d: vmlist_d.remove('') for r in vmlist: if is_on_the_bridge(r, args.brname): LOG.Info("===========================") LOG.Info("Destroying : %s" % r) cmds = [] # Only destroy the vms here cmds.append("sudo virsh destroy %s || true" % r) rv, rs = CMD.shell(";".join(cmds)) LOG.Info("Return Value: %s" % str(rv)) for rl in rs: LOG.Info(rl) if args.delete_all: LOG.Info("!!!!!!!!!!!!!!!!!!!!!!!!!!!") LOG.Info("Vms will be perminantly deleted.") for r in vmlist + vmlist_d: if is_on_the_bridge(r, args.brname): LOG.Info("===========================")
def __init__(self, msg, check_func, interval=10, slowdown=1): LOG.Info("Waiting for " + msg) self.check_func = check_func self.interval = interval self.slowdown = slowdown
def stx_apply_application(node0, helm_charts): LOG.Info("Applying application with helm charts %s." % helm_charts) # copy the helm_charts to test folder and upload it to the target. charts_filename = os.path.basename(helm_charts) appname = os.path.splitext(charts_filename)[0] node0.copy_to_node(helm_charts, "~/") failures = 0 appstat = __stx_app_stat(node0, appname)[0] def handle_failure(failures): failures += 1 get_pod_logs(node0, logname="pod_logs_failed_%d" % failures) return failures while appstat != APP_APPLY_SUCCESS: LOG.Info("App %s current status: %s" % (appname, appstat)) if appstat == APP_NONE: retval, retlog = __stx_app_op(node0, APP_OP_UPLOAD, helm_charts=charts_filename) real_app_name = None for l in retlog: if l.find(" name ") >= 0: real_app_name = l.split("|")[2].strip() if not real_app_name: __exit_with_exception() LOG.Info("App Real Name: %s" % real_app_name) appname = real_app_name appstat = wait_for_app_status_finish(node0, appname, 20, APP_UPLOADING) elif appstat == APP_UPLOAD_SUCCESS: retval, retlog = __stx_app_op(node0, APP_OP_APPLY, appname=appname) appstat = wait_for_app_status_finish(node0, appname, 120, APP_APPLYING) elif appstat == APP_UPLOAD_FAILED: failures = handle_failure(failures) retval, retlog = __stx_app_op(node0, APP_OP_DELETE, appname=appname) appstat = wait_for_app_status_finish(node0, appname, 20, APP_DELETING) elif appstat == APP_APPLY_SUCCESS: # should not run here break elif appstat == APP_APPLY_FAILED: failures = handle_failure(failures) retval, retlog = __stx_app_op(node0, APP_OP_REMOVE, appname=appname) appstat = wait_for_app_status_finish(node0, appname, 30, APP_REMOVING) elif appstat == APP_REMOVE_FAILED: failures = handle_failure(failures) retval, retlog = __stx_app_op(node0, APP_OP_REMOVE, appname=appname) appstat = wait_for_app_status_finish(node0, appname, 30, APP_REMOVING) elif appstat == APP_DELETE_FAILED: # Fatal LOG.Error("Failed to delete app %s after %d failures" % (appname, failures)) __exit_with_exception() elif appstat in [ APP_UPLOADING, APP_APPLYING, APP_REMOVING, APP_DELETING ]: failures = handle_failure(failures) appstat = wait_for_app_status_finish(node0, appname, 20, appstat) else: # Fatal LOG.Error("Unknown application status %s" % appstat) __exit_with_exception() if failures >= 5: # Fatal LOG.Error("Application %s apply failed!" % app_name) __exit_with_exception() return appname