def close_pxe_services():
"""
"""
services = ['dnsmasq']
for service in services:
LOG.Info("Stopping %s ..." % service)
CMD.shell('sudo systemctl stop {}'.format(service))
def __init__(self, pxe_server_config_json, iso, default_install=0):
'''
pxe_server_config_json: json file to describe pxe server.
iso: ISO file to install.
default_install: install mode for uefi mode
0: boot from hard disk
1: standard controller
2: all-in-one controller
3: all-in-one controoler (low-latency)
'''
CONFIG = UTILS.load_json_config(pxe_server_config_json)
CONFIG["iso"] = os.path.abspath(iso)
CONFIG["default_install"] = default_install
for tag in ["tftp_dir", "mnt_point", "http_root"]:
CONFIG[tag] = os.path.abspath(CONFIG[tag])
self.iso = CONFIG["iso"]
self.iso_name = os.path.basename(self.iso).replace('.iso', '')
self.tftp_dir = CONFIG["tftp_dir"]
self.mnt_point = CONFIG["mnt_point"]
self.http_server = CONFIG["http_server"]
self.http_mnt_point = os.path.join(CONFIG["http_root"], CONFIG["prj_name"])
self.prj_name = CONFIG["prj_name"]
self.default_install = CONFIG["default_install"]
self.CONFIG = CONFIG
CMD.shell("sudo mkdir -p %s" % self.http_mnt_point)
def check_pxe_services():
"""
"""
services = ['dnsmasq']
for service in services:
LOG.Info("Restarting %s ..." % service)
CMD.shell('sudo systemctl restart {}'.format(service))
def add_sshkey_config(server, user):
ssh_config = "%s/.ssh/config" % os.environ['HOME']
if os.path.exists(ssh_config):
retval, retlog = CMD.shell("grep \"%s\" %s" % (server, ssh_config),
silent=True)
if retval == 0:
LOG.Warning("%s already configured in .ssh/config." % server)
return
config_ssh = []
config_ssh.append("Host %s" % server)
config_ssh.append(" IdentityFile %s" % TEST_KEY)
config_ssh.append(" User %s" % user)
for config in config_ssh:
CMD.shell("echo \"%s\" >> %s" % (config, ssh_config))
def scp_from_server(src,
dst,
server,
user,
password=None,
cwd=None,
logfile=None,
silent=True):
scpcmd = "scp -r -oStrictHostKeyChecking=no -oCheckHostIP=no " \
"%s@%s:%s %s" % (user, server, src, dst)
if password:
retval, retlog = run_expect_cmd_with_password(scpcmd,
password,
cwd=cwd,
logfile=logfile,
silent=silent)
else:
retval, retlog = CMD.shell(scpcmd,
cwd=cwd,
logfile=logfile,
silent=silent)
for l in retlog:
if l.find("No such file") >= 0:
retval = 1
remove_ssh_log_head(retlog)
return retval, retlog
def get_system_logs(node):
syslog_folder = getlogfile("system_logs")
CMD.shell("mkdir -p %s/" % syslog_folder, silent=True)
ret, retlog = node.ssh("collect -a", forcesudo=True, silent=True)
if ret != 0:
LOG.Warning("collect returned error code: %s" % str(ret))
LOG.print_warning(retlog)
node.copy_from_node("/scratch/*.tgz", syslog_folder)
node.ssh('sudo sm-dump',
logfile=getlogfile("sm-dump.log"),
sudo=True,
silent=True)
LOG.Info("Please check the full system log under %s." % syslog_folder)
def run_expect_cmd_with_password(cmd,
password,
cwd=None,
logfile=None,
silent=False):
ecmds = []
ecmds.append("spawn %s" % cmd)
ecmds.append("expect {")
ecmds.append(" \"*assword:*\" {")
ecmds.append(" send \"%s\\r\"" % password)
ecmds.append(" expect \"*$ \"")
ecmds.append(" }")
ecmds.append(" \"*password for sysadmin:*\" {")
ecmds.append(" send \"%s\\r\"" % password)
ecmds.append(" expect \"*$ \"")
ecmds.append(" }")
ecmds.append(" \"*$ \"")
ecmds.append("}")
ecmds.append("lassign [wait] pid spawnid os_error_flag value")
ecmds.append("exit $value")
# LOG.print_log(ecmds)
retval, retlog = CMD.shell("expect -c '\n"
"%s\n"
"' 2>&1" % "\n".join(ecmds),
cwd=cwd,
logfile=logfile,
silent=silent)
return retval, retlog
def cmdhost(cmd, cwd=None, logfile=None, silent=False):
retval, result = CMD.shell(cmd, cwd=cwd, logfile=logfile, silent=silent, DEBUG=DEBUG)
if not silent:
LOG.Info("Finished \"%s\"\n" % cmd, silent=silent)
LOG.Info("\n".join(result))
return retval
def secure_ssh(cmd,
server,
user,
password=None,
cwd=None,
logfile=None,
silent=False):
if not silent:
LOG.Info("EXEC >>> %s@%s: \"%s\"" % (user, server, cmd))
head = "ssh -t -oStrictHostKeyChecking=no -oCheckHostIP=no " \
"%s@%s" % (user, server)
if password:
sshcmd = "%s %s" % (head, cmd)
retval, retlog = run_expect_cmd_with_password(sshcmd,
password,
cwd=cwd,
logfile=logfile,
silent=silent)
for l in retlog:
if l.find("spawn id exp4 not open") >= 0:
retval = 1
break
else:
sshcmd = "%s 'echo \"%s\"; %s' 2>&1" % (head, SSH_LOG_FLAG, cmd)
retval, retlog = CMD.shell(sshcmd,
cwd=cwd,
logfile=logfile,
silent=silent)
remove_ssh_log_head(retlog)
return retval, retlog
def add_route_for_oam(self, oam_gate_way):
oam_ip = self.get_oam_ip()
oam_bridge = self.get_oam_bridge()
if oam_ip and oam_bridge:
LOG.Info("Add route for oam %s via %s on dev %s" %
(oam_ip, oam_gate_way, oam_bridge))
ret, log = CMD.shell("sudo ip route add %s via %s dev %s" %
(oam_ip, oam_gate_way, oam_bridge))
if ret:
LOG.print_error(log)
def __mount_point(self, mnt_point, iso):
if os.listdir(mnt_point):
LOG.Info('{} is already mounted, umounting'.format(mnt_point))
self.__umount_point(mnt_point)
ret, log = CMD.shell('sudo mount {0} {1}'.format(self.iso, mnt_point))
if ret == 0:
LOG.Info('ISO mounted on {}'.format(mnt_point))
else:
__print_errorlog("MOUNT", log)
exit(1)
def kvm_update_netmap(self):
retval, mac_list = CMD.shell(
"sudo virsh domiflist %s | grep bridge | awk '{print $NF}'" %
self.name,
silent=True)
retval, br_list = CMD.shell(
"sudo virsh domiflist %s | grep bridge | awk '{print $(NF-2)}'" %
self.name,
silent=True)
nic_list = ['default', 'mgmt', 'data0', 'data1']
if retval == 0:
self.CONFIG['num_of_nic'] = len(mac_list)
for i in range(len(mac_list)):
self.Info("%s: %s %s" % (nic_list[i], mac_list[i], br_list[i]))
self.__update_nic_info(nic_list[i], mac_list[i], br_list[i])
return True
else:
self.Error("Failed to get Network Map.")
return False
def cleanup_network_env(self, oam_gate_way):
default_nic = None
for n in self.CONFIG['network'].keys():
if n == 'default':
default_nic = self.CONFIG['network'][n]
continue
nic = self.CONFIG['network'][n]
if 'brname' in nic and nic['brname']:
ret, log = CMD.shell("sudo ip addr | grep %s | grep %s" %
(oam_gate_way, nic['brname']),
silent=True)
if ret == 0:
LOG.Info("Delete %s on %s." %
(oam_gate_way, nic['brname']))
ret, log = CMD.shell("sudo ip addr del %s dev %s" %
(oam_gate_way, nic['brname']))
def default_route_on_nic(nicname):
ret, log = self.ssh('sudo route | grep default | grep %s' %
nicname,
sudo=True,
silent=True)
if ret == 0:
LOG.Info("Already has default route on %s" % nicname)
return ret == 0
for n in self.CONFIG['network'].keys():
if n == 'default':
continue
nic = self.CONFIG['network'][n]
if default_route_on_nic(nic['name']):
LOG.Info("Delete default route on %s." % nic['name'])
self.ssh('sudo route del default', sudo=True)
if default_nic and not default_route_on_nic(default_nic['name']):
LOG.Info("Add default route %s on %s" %
(oam_gate_way, default_nic['name']))
self.ssh('sudo route add default gw %s %s' %
(oam_gate_way, default_nic['name']),
sudo=True)
def prepare_for_node(self, node):
cmds = []
# Special grub.cfg for the node
cmds.append("sudo cp %s/grub.cfg.bak %s/grub.cfg" % (
self.tftp_dir, self.tftp_dir))
# Modify boot_device and rootfs_device
cmds.append("sudo sed -i -e \"s,boot_device=sda,boot_device=%s,\" %s/grub.cfg" % (
node.get_boot_device(), self.tftp_dir))
cmds.append("sudo sed -i -e \"s,rootfs_device=sda,rootfs_device=%s,\" %s/grub.cfg" % (
node.get_rootfs_device(), self.tftp_dir))
# Special pxeboot.cfg for the node
cmds.append("sudo cp %s/pxeboot.cfg.bak %s/pxeboot.cfg" % (
self.tftp_dir, self.tftp_dir))
# Modify boot_device and rootfs_device
cmds.append("sudo sed -i -e \"s,boot_device=sda,boot_device=%s,\" %s/pxeboot.cfg" % (
node.get_boot_device(), self.tftp_dir))
cmds.append("sudo sed -i -e \"s,rootfs_device=sda,rootfs_device=%s,\" %s/pxeboot.cfg" % (
node.get_rootfs_device(), self.tftp_dir))
#####
CMD.shell("; ".join(cmds))
def __ipmi_cmd(self, ipmi_cmd, silent=False):
cmd = 'ipmitool -I lanplus -H %s -U %s -P %s -p %s %s' \
% (self.bmc_ip,
self.bmc_user,
self.bmc_pwd,
self.bmc_port,
ipmi_cmd)
self.Info("IPMICMD: %s" % cmd)
ret, log = CMD.shell(cmd, silent=silent)
if not silent:
self.__check_ipmi_result(ret, log)
time.sleep(3)
return ret, log
def remove_from_knownhosts(self):
oam_ip = self.get_oam_ip()
floating_ip = self.get_floating_ip()
cmd = []
if oam_ip:
cmd.append(
'ssh-keygen -f "$HOME/.ssh/known_hosts" -R %s >/dev/null 2>&1'
% oam_ip)
if floating_ip:
cmd.append(
'ssh-keygen -f "$HOME/.ssh/known_hosts" -R %s >/dev/null 2>&1'
% floating_ip)
retval, retlog = CMD.shell(';'.join(cmd), silent=True)
if retlog:
LOG.print_log(retlog)
def mount_iso(self):
"""Mounting ISO and prepare pxe tftp server"""
## clean up test environment
http_root = self.__remove_folder(
os.path.join(self.http_mnt_point, self.iso_name))
tftp_dir = self.__remove_folder(self.tftp_dir)
## Setup pxe install folder on tftp server
self.__mount_point(self.mnt_point, self.iso)
CMD.shell(
"sudo %s/pxeboot_setup.sh "
"-u http://%s/%s/%s "
"-t %s" % (
self.mnt_point,
self.http_server, self.prj_name, self.iso_name,
self.tftp_dir
))
self.__umount_point(self.mnt_point)
cmds = []
cmds.append("sudo chmod 777 %s" % self.tftp_dir)
######## MODIFY INSTALL LABEL FOR UEFI BOOT
# Modify default install option
cmds.append("sudo sed -i -e \"s,default=0,default=%s,\" %s/grub.cfg" % (
str(self.default_install), self.tftp_dir))
# Modify timeout
cmds.append("sudo sed -i -e \"s,timeout=10,timeout=3,\" %s/grub.cfg" % self.tftp_dir)
# Modify console (always use graphic console)
cmds.append("sudo sed -i -e \"s,serial console=ttyS0\\,115200n8,console=tty0,\" "
"%s/grub.cfg" % (self.tftp_dir))
# Backup grub.cfg
cmds.append("sudo cp %s/grub.cfg %s/grub.cfg.bak" % (
self.tftp_dir, self.tftp_dir))
######## MODIFY INSTALL LABEL FOR BIOS BOOT
# Modify default install option
cmds.append("sudo sed -i -e \"s,DEFAULT menu\\.c32,DEFAULT %s,\" %s/pxeboot.cfg" % (
str(self.__get_bios_install_label()), self.tftp_dir))
# Modify timeout
cmds.append("sudo sed -i -e \"s,TIMEOUT 100,TIMEOUT 3,\" %s/pxeboot.cfg" % self.tftp_dir)
# Backup pxeboot.cfg
cmds.append("sudo cp %s/pxeboot.cfg %s/pxeboot.cfg.bak" % (
self.tftp_dir, self.tftp_dir))
# No change for the default password
cmds.append("sed -i 's/chage -d 0 sysadmin/#chage -d 0 sysadmin/g' "
"`grep \"chage -d 0 sysadmin\" %s -rl `" % self.tftp_dir)
# Symbol link to http server
cmds.append("sudo ln -s %s %s" % (self.tftp_dir, http_root))
#####
CMD.shell("; ".join(cmds))
## PXE server boot_file set as grubx64.efi
CMD.shell("sudo cp EFI/grubx64.efi grubx64.efi", cwd=self.tftp_dir)
def create_secure_path(server, user, password):
cmd = "ssh -t -oStrictHostKeyChecking=no -oCheckHostIP=no " \
"%s@%s mkdir -p ~/.ssh" % (user, server)
retval, retlog = run_expect_cmd_with_password(cmd, password)
if retval != 0:
return retval, retlog
if not os.path.exists(TEST_PUBKEY):
LOG.Info("pubkey %s not existing, generate new key" % TEST_PUBKEY)
ret, log = CMD.shell("ssh-keygen -o -f %s -P \"\"" % TEST_KEY)
LOG.print_log(log)
add_sshkey_config(server, user)
retval, retlog = scp_to_server(TEST_PUBKEY, "~/.ssh/authorized_keys",
server, user, password)
return retval, retlog
def __umount_point(self, mnt_point):
umounting_attempts = 3
while umounting_attempts > 0:
ret, log = CMD.shell('sudo umount -l {}'.format(mnt_point))
if ret != 0 and umounting_attempts:
LOG.Info('Failed to umount {}, retrying...'.format(
mnt_point))
elif ret != 0 and not umounting_attempts:
LOG.Error('Max umounting attempts reached, leaving '
'installation')
__print_errorlog("UMOUNT", log)
exit(1)
else:
break
umounting_attempts -= 1
def __remove_folder(self, folder):
if os.path.exists(folder):
CMD.shell("sudo rm -rf %s" % folder)
# shutil.rmtree(folder)
return folder
def check():
return 0 == CMD.shell(pingcmd, silent=True)[0]
def power_off(self):
if self.is_power_on():
return CMD.shell("sudo virsh destroy %s" % self.name)
return 0, []
def __create_folder(self, folder):
self.__remove_folder(folder)
CMD.shell("sudo mkdir -p %s" % folder)
# os.makedirs(folder)
return folder
def power_on(self):
if not self.is_power_on():
return CMD.shell("sudo virsh start %s" % self.name)
return 0, []
def reset(self):
if self.is_power_on():
retval, retlog = CMD.shell("sudo virsh destroy %s" % self.name)
if retval != 0:
return retval, retlog
return CMD.shell("sudo virsh start %s" % self.name)
def check_ping(self):
if self.get_oam_ip():
pingcmd = "ping %s -c 1 >/dev/null 2>&1" % self.get_oam_ip()
return 0 == CMD.shell(pingcmd, silent=True)[0]
return False
type=str,
default='virbr1')
args = parser.parse_args()
def is_on_the_bridge(domID, brname):
rv, brlist = CMD.shell("sudo virsh domiflist %s | grep bridge | awk '{print $3;}'" % domID)
if brname in brlist:
return True
return False
vmlist = []
vmlist_d = []
if args.vmname:
cmd = "sudo virsh list --all | grep %s | awk '{print $2}'" % args.vmname
retval, vmlist = CMD.shell(cmd)
else:
cmd = "sudo virsh list --all | grep running | awk '{print $2}'"
retval, vmlist = CMD.shell(cmd)
cmd = "sudo virsh list --all | grep -v running | grep -v Name | awk '{print $2}'"
retval, vmlist_d = CMD.shell(cmd)
while '' in vmlist_d:
vmlist_d.remove('')
for r in vmlist:
if is_on_the_bridge(r, args.brname):
LOG.Info("===========================")
LOG.Info("Destroying : %s" % r)
cmds = []
# Only destroy the vms here
cmds.append("sudo virsh destroy %s || true" % r)
def is_on_the_bridge(domID, brname):
rv, brlist = CMD.shell("sudo virsh domiflist %s | grep bridge | awk '{print $3;}'" % domID)
if brname in brlist:
return True
return False
def is_power_on(self):
retval, retlog = CMD.shell("sudo virsh list | grep %s" % self.name,
silent=True)
return retval == 0
