def run_fab_commands2(test_obj, handle, prompt):
test_conf = test_obj.argument['test_conf']['test_conf']
testbed_config = test_obj.argument['testbed_config']['testbed_config']
cmd = "cd /opt/contrail/utils"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
cmd = "fab install_storage"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 1200)
if re.search('Abort', output):
gen_lib.Print("ERROR: %s Aborted" % cmd)
test_obj.argument['err_msg'] += "ERROR: fab install_storage failed.\n"
sys.exit()
ntp_update_all_nodes(test_obj, handle, prompt)
cmd = "fab setup_storage"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 1200)
if re.search('Abort', output):
gen_lib.Print("ERROR: %s Aborted" % cmd)
test_obj.argument['err_msg'] += "ERROR: fab setup_storage failed.\n"
sys.exit()
#time.sleep(30)
#check_ceph_status(test_obj,handle,prompt)
return
def upgrade_kernel(test_obj,
handle,
prompt,
setup_data_interface,
host_list=None):
test_conf = test_obj.argument['test_conf']['test_conf']
testbed_config = test_obj.argument['testbed_config']['testbed_config']
profile_name = test_conf['profile_name']
fab_node_host = testbed_config['%s,fab_node' % profile_name]
fab_node = testbed_config['%s,node_name' % fab_node_host]
#if test_conf['ubuntu_version'] == "14.04" :
# return False
cmd = "cd /opt/contrail/utils"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
cmd = "fab upgrade_kernel_all"
output = gen_lib.send_cmd(test_obj, handle, cmd, [
'to boot with new kernel version',
'kernel is already of expected version'
], 1800)
time.sleep(600)
contrail_branch = re.search('contrail-install-packages_(\d\.\d+)',
test_conf['install_packages'][0]).group(1)
if test_conf['ubuntu_version'] == "12.04":
exp_kernel_version = test_conf['12.04,kernel_upgrade,version']
elif test_conf['ubuntu_version'] == "14.04" and contrail_branch <= "2.0":
exp_kernel_version = test_conf['14.04,kernel_upgrade,version,2.0']
else:
exp_kernel_version = test_conf['14.04,kernel_upgrade,version,2.10']
if host_list == None:
host_list = testbed_config['%s,hosts_list' % profile_name]
for host in host_list:
node = testbed_config['%s,node_name' % host]
cmd = "uname -a"
test_obj.argument['%s_handle' % node] = test_obj.create_ssh_handle(
node_name=node, ntp_update=True, run_time=test_conf['runtime'])
output = gen_lib.send_cmd(test_obj,
test_obj.argument['%s_handle' % node], cmd,
testbed_config['%s,prompt' % node], 10)
if re.search(exp_kernel_version, output):
msg = "INFO: %s kernel upgrade done\n" % node
gen_lib.Print(msg)
else:
err_msg = "ERROR: %s kernel upgrade failed\n" % node
gen_lib.Print(err_msg)
test_obj.argument['err_msg'] += err_msg
handle = test_obj.argument['%s_handle' % fab_node]
if setup_data_interface:
fab_setup_interface(test_obj, handle, prompt)
return True
def live_migrate(test_obj, handle, prompt):
testbed_config = test_obj.argument['testbed_config']['testbed_config']
test_conf = test_obj.argument['test_conf']['test_conf']
domain_name = "englab.juniper.net"
vm_name = "VM1"
vm_id = ostack.get_vm_id(test_obj, handle, prompt, vm_name)
current_hostname1 = ostack.get_vm_hostinfo(test_obj, handle, prompt,
vm_name)
current_hostname1 = current_hostname1.rsplit("." + domain_name)[0]
new_host = ""
compute_host_list = ceph.get_storage_host_list(test_obj, handle, prompt)
print compute_host_list, current_hostname1
for host in compute_host_list:
host = host.rsplit("." + domain_name)[0]
print host, current_hostname1
if host != current_hostname1:
new_host = host
break
print "Current host:", current_hostname1
print "New host:", new_host
ostack.live_migrate_vm(test_obj, handle, prompt, vm_id, new_host)
time.sleep(60)
args = {}
args['handle'] = handle
args['prompt'] = prompt
args['test_obj'] = test_obj
args['vm_name'] = "VM1"
args['status'] = "ACTIVE"
args['power_state'] = "Running"
args['timeout'] = 600
ostack.wait_until_vm_status(args)
current_hostname2 = ostack.get_vm_hostinfo(test_obj, handle, prompt,
vm_name)
current_hostname2 = current_hostname2.rsplit("." + domain_name)[0]
new_host = new_host.rsplit("." + domain_name)[0]
if current_hostname2 == new_host:
gen_lib.Print("PASS: Live-migration working...")
else:
test_obj.argument[
'err_msg'] += "ERROR: Live-migration is not working...\n"
gen_lib.Print("FAIL: Live-migration is not working...")
def cleanup(self):
for h in self.argument['telnet_handles']:
try:
h.close()
except:
traceback.print_exc()
gen_lib.Print("DEBUG: error closing telnet handle")
for h in self.argument['ssh_handles']:
try:
h.logout()
except:
traceback.print_exc()
gen_lib.Print("DEBUG: error closing ssh handle")
def reboot_mfc(handle, prompt):
handle.PROMPT = prompt
cmd = "reload"
handle.sendline(cmd)
i = handle.expect(
[prompt, pexpect.EOF, pexpect.TIMEOUT, 'save first', 'Confirm reload'],
timeout=10)
output = handle.before
print output
if i == 3 or i == 4:
cmd = "yes"
handle.sendline(cmd)
i = handle.expect(
[prompt, pexpect.EOF, pexpect.TIMEOUT, 'Confirm reload'],
timeout=10)
output = handle.before
print output
if i == 3:
cmd = "yes"
handle.sendline(cmd)
i = handle.expect(
[prompt, pexpect.EOF, pexpect.TIMEOUT, 'Confirm reload'],
timeout=10)
output = handle.before
print output
if i == 0:
gen_lib.Print("INFO: vm rebooted successfully")
else:
msg = "ERROR: vm did not reboot correctly"
test_obj.argument['err_msg'] += msg
def provision_servers(self):
smgr_handle = self.arg['handle']
smgr_prompt = self.arg['prompt']
test_obj = self.arg['test_obj']
test_conf = test_obj.argument['test_conf']['test_conf']
testbed_config = test_obj.argument['testbed_config']['testbed_config']
cluster_id = get_cluster_id(testbed_config['cluster_json_file'])
cmd = test_conf[
'smgr_client_cmd'] + " provision --cluster_id %s contrail_test_pkg" % cluster_id
print "CMD: ", cmd
smgr_handle.sendline(cmd)
i = smgr_handle.expect(['y/N', pexpect.EOF, pexpect.TIMEOUT],
timeout=120)
cmd = 'y'
output = gen_lib.send_cmd(test_obj, smgr_handle, cmd, smgr_prompt, 120)
if not re.search('provision issued', output):
err_msg = "ERROR: servers not provisioned..\n"
gen_lib.Print(err_msg)
test_obj.argument['err_msg'] += err_msg
sys.exit()
self.wait_until_provision_complete(smgr_handle, smgr_prompt)
time.sleep(180)
return
def re_image_all(self):
handle = self.arg['handle']
prompt = self.arg['prompt']
test_obj = self.arg['test_obj']
test_conf = test_obj.argument['test_conf']['test_conf']
testbed_config = test_obj.argument['testbed_config']['testbed_config']
cluster_id = get_cluster_id(testbed_config['cluster_json_file'])
cmd = test_conf[
'smgr_client_cmd'] + " reimage --cluster_id %s ubuntu-server1" % cluster_id
print "CMD: ", cmd
handle.sendline(cmd)
i = handle.expect(['y/N', pexpect.EOF, pexpect.TIMEOUT], timeout=120)
cmd = 'y'
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 60)
if re.search('No Image', output):
err_msg = "ERROR: image-not found error seen\n"
gen_lib.Print(err_msg)
test_obj.argument['err_msg'] += err_msg
sys.exit()
ret = re.search('server\(s\) reimage queued|issued', output)
if not ret:
err_msg = "ERROR: server re-image command failed.\n"
print err_msg
test_obj.argument['err_msg'] += err_msg
sys.exit()
self.wait_until_reimage_complete()
time.sleep(180) # to allow servers to reboot.
def PostResult(self):
if self.argument['err_msg'] == "":
test_result = "PASS"
else:
test_result = "FAIL"
gen_lib.Print("TEST RESULT: %s %s\n" % (sys.argv[0], test_result))
def get_data_interface_name(test_obj,handle,prompt,node_name):
testbed_config = test_obj.argument['testbed_config']['testbed_config']
cmd = "/sbin/ifconfig -a | grep '%s'" %testbed_config['%s,data_link,mac'%node_name]
output = gen_lib.send_cmd(test_obj,handle,cmd,prompt,10)
gen_lib.Print("####")
gen_lib.Print(output)
gen_lib.Print("####")
ret = re.search('eth\d',output)
iface = ret.group(0)
#ip = testbed_config['%s,data_link,ip'%node_name]
#netmask = testbed_config['%s,data_link,netmask'%node_name]
#cmd = "/sbin/ifconfig %s %s netmask %s up"%(iface,ip,netmask)
#output = gen_lib.send_cmd(test_obj,handle,cmd,prompt,10)
#gen_lib.Print(output)
return iface
def clear_pxe_new_page(mfc_tn):
gen_lib.Print("PXE menu moving to next page. So clear the data")
mfc_tn.read_until(".*",2)
mfc_tn.write("[B")
gen_lib.Print("==========")
gen_lib.Print(mfc_tn.read_until("nopattern",2))
gen_lib.Print("==========")
mfc_tn.read_until(".*",2)
mfc_tn.write("[A")
image_2_1_out = mfc_tn.read_until("nopattern",2)
mfc_tn.write("[B")
mfc_tn.read_until(".*",2)
mfc_tn.write("[B")
image_2_3_out = mfc_tn.read_until("nopattern",2)
mfc_tn.write("[A")
mfc_tn.read_until(".*",2)
mfc_tn.write("[A")
mfc_tn.read_until(".*",2)
return image_2_1_out,image_2_3_out
def PowerCycle(handle,menu_key):
if menu_key == '^' :
handle.write(" ")
else :
#handle.write("")
handle.write(chr(26))
out = handle.read_until("close current connection to port",10)
gen_lib.Print(out)
#ret = re.search('(\S)\s*reboot device using power-switch',out)
#reboot_keystroke = ret.group(1)
reboot_keystroke = 'r'
handle.write(reboot_keystroke)
out = handle.read_until("Are you sure you want to REBOOT this port",5)
gen_lib.Print(out)
if(not re.search("Are you sure you want to REBOOT this port",out,re.M)): return "FAILED"
handle.write("y")
out = handle.read_until("Rebooting",10)
gen_lib.Print(out)
def check_disk_list(test_obj, handle, prompt):
cmd = "show media-cache disk list"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 20)
ret = re.findall('dc_\d+\s*SAS', output)
if len(ret) != 2:
err_msg = "ERROR: disk did not attach correctly to MFC\n"
gen_lib.Print(err_msg)
test_obj.argument['err_msg'] += err_msg
def ExecuteSetupStorage(test_obj, handle, prompt):
cmd = "cd /opt/contrail/contrail_packages"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
cmd = "./setup_storage.sh"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 300)
if re.search('Abort', output):
gen_lib.Print("ERROR: %s Aborted" % cmd)
test_obj.argument['err_msg'] += "ERROR: ./setup_storage.sh failed.\n"
sys.exit()
def update_testbed_file(test_obj, fab_node_handle, testbed_file, info):
testbed_config = test_obj.argument['testbed_config']['testbed_config']
test_conf = test_obj.argument['test_conf']['test_conf']
src_ip = testbed_config['regression,ip']
src_login = testbed_config['regression,login']
src_passwd = testbed_config['regression,password']
profile_name = test_conf['profile_name']
fab_node_host = testbed_config['%s,fab_node' % profile_name]
fab_node = testbed_config['%s,node_name' % fab_node_host]
d, f = os.path.split(testbed_file)
for host in testbed_config['%s,hosts_list' % profile_name]:
node = testbed_config['%s,node_name' % host]
test_obj.argument['%s_data_iface' %
node] = os_lib.get_data_interface_name(
test_obj, test_obj.argument['%s_handle' % node],
testbed_config['%s,prompt' % node], node)
cmd = "cp %s /tmp/" % testbed_file
gen_lib.Print(cmd)
output = commands.getoutput(cmd)
gen_lib.Print(output)
for host in testbed_config['%s,hosts_list' % profile_name]:
node = testbed_config['%s,node_name' % host]
iface = test_obj.argument['%s_data_iface' % node]
cmd = "sed -i 's/%s_data_interface/%s/' /tmp/%s " % (node, iface, f)
gen_lib.Print(cmd)
output = commands.getoutput(cmd)
gen_lib.Print(output)
src_file = "/tmp/%s" % f
dest = '/opt/contrail/utils/fabfile/testbeds/testbed.py'
test_obj.remote_scp_from(fab_node_handle,
testbed_config['%s,prompt' % fab_node], src_file,
src_ip, src_login, src_passwd, dest)
def run_fab_commands1(test_obj,
handle,
prompt,
setup_data_interface=False,
host_list=None):
test_conf = test_obj.argument['test_conf']['test_conf']
testbed_config = test_obj.argument['testbed_config']['testbed_config']
profile_name = test_conf['profile_name']
fab_node_host = testbed_config['%s,fab_node' % profile_name]
fab_node = testbed_config['%s,node_name' % fab_node_host]
node_rebooted = upgrade_kernel(test_obj, handle, prompt,
setup_data_interface, host_list)
if node_rebooted:
handle = test_obj.argument['%s_handle' % fab_node]
cmd = "cd /opt/contrail/utils"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
cmd = "fab install_contrail"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 3600)
if re.search('Abort', output):
gen_lib.Print("ERROR: %s Aborted" % cmd)
test_obj.argument['err_msg'] += "ERROR: fab install_contrail failed.\n"
sys.exit()
cmd = "cd /opt/contrail/utils"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
cmd = "fab setup_all"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 1800)
if re.search('Abort', output):
gen_lib.Print("ERROR: %s Aborted" % cmd)
test_obj.argument['err_msg'] += "ERROR: fab setup_all falied.\n"
sys.exit()
def setup_storage(test_obj, handle, prompt):
test_conf = test_obj.argument['test_conf']['test_conf']
testbed_config = test_obj.argument['testbed_config']['testbed_config']
cmd = "cd /opt/contrail/utils"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
cmd = "fab setup_storage"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 1200)
if re.search('Abort', output):
gen_lib.Print("ERROR: %s Aborted" % cmd)
test_obj.argument['err_msg'] += "ERROR: fab setup_storage failed.\n"
sys.exit()
def remote_scp(self, handle, prompt, src_file, cmd, passwd, scp_timeout):
handle.expect('.*', timeout=1)
gen_lib.Print(handle.before)
gen_lib.Print("CMD:" + cmd)
handle.sendline(cmd)
ret = handle.expect(["\?", "password:"******"yes")
ret = handle.expect(["password:"******"Unknown prompt received: " + output)
if (re.search("100%", output, re.M)):
gen_lib.Print("INFO: File : %s copied Successfully" % src_file)
return True
else:
gen_lib.Print("ERROR: File : %s copy failed" % src_file)
return False
def configure_livem(test_obj, handle, prompt, fab_node):
testbed_config = test_obj.argument['testbed_config']['testbed_config']
test_conf = test_obj.argument['test_conf']['test_conf']
cmd = "mkdir /store"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
src_ip = testbed_config['regression,ip']
src_login = testbed_config['regression,login']
src_passwd = testbed_config['regression,password']
src_file = test_conf['ceph_nfs_image_src']
dest = "/store"
status = test_obj.remote_scp_from(handle,
testbed_config['%s,prompt' % fab_node],
src_file, src_ip, src_login, src_passwd,
dest)
if not status:
test_obj.argument[
'err_msg'] += "test_obj.remote_scp for %s failed" % src_file
sys.exit()
cmd = "cd /store"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
d, f = os.path.split(test_conf['ceph_nfs_image_src'])
cmd = "rm -rf livemnfs.qcow2"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 300)
cmd = "gunzip %s" % f
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 300)
cmd = "cd /opt/contrail/utils"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
live_migration_enabled = False
#args = {}
#args['handle'] = handle
#args['prompt'] = prompt
#args['tenant_name'] = 'admin'
#args['tenant_id'] = ostack.get_tenant_id(args)
#args['quota'] = 1600
#ostack.set_cinder_quota(args)
for i in xrange(3):
cmd = "fab setup_nfs_livem_global"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 900)
if re.search('Abort', output):
test_obj.argument[
'warn_msg'] += "WARNING: fab setup_nfs_livem_global aborted and re-tried"
gen_lib.Print("ERROR: %s Aborted" % cmd)
cmd = "umount /var/lib/nova/instances/global"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 60)
else:
live_migration_enabled = True
break
if not live_migration_enabled:
test_obj.argument[
'err_msg'] += "ERROR: fab setup_nfs_live_migration aborted more than once"
sys.exit()
def fix_telnet_login(test_obj,handle,prompt):
src_ip = "10.87.129.2"
src_login = "******"
src_passwd = "n1keenA"
src_file = "/tmp/telnet_0.17-36build1_amd64.deb"
dest = src_file
test_obj.remote_scp(handle,prompt,src_ip,src_login,src_passwd,src_file,dest)
cmd = "dpkg -i /tmp/telnet_0.17-36build1_amd64.deb"
handle.write(cmd)
gen_lib.Print(handle.read_until(prompt,120))
#src_list = [ 'deb http://cz.archive.ubuntu.com/ubuntu lucid main universe']
#cmd = "\cp -f /etc/apt/sources.list /etc/apt/sources.list.bk\n"
#handle.write(cmd)
#gen_lib.Print(handle.read_until(prompt,5))
#cmd = "echo > /etc/apt/sources.list\n"
#handle.write(cmd)
#gen_lib.Print(handle.read_until(prompt,5))
#for src in src_list :
# cmd = "echo \"" + src + "\"" + " >> /etc/apt/sources.list\n"
# handle.write(cmd)
# gen_lib.Print(handle.read_until(prompt,5))
#cmd = "apt-get update\n"
#handle.write(cmd)
#gen_lib.Print(handle.read_until(prompt,600))
#cmd = "apt-get install xinetd telnetd\n"
#handle.write(cmd)
#gen_lib.Print(handle.read_until('Do you want to continue',30))
#handle.write('Y\n')
#gen_lib.Print(handle.read_until(prompt,100))
cmd = """# Simple configuration file for xinetd
#
# Some defaults, and include /etc/xinetd.d/
defaults
{
# Please note that you need a log_type line to be able to use log_on_success
# and log_on_failure. The default is the following :
# log_type = SYSLOG daemon info
instances = 60
log_type = SYSLOG authpriv
log_on_success = HOST PID
log_on_failure = HOST
cps = 25 30
}
"""
handle.write("echo \"" + cmd + " \" > /etc/xinetd.conf\n")
gen_lib.Print(handle.read_until(prompt,30))
handle.write("echo \"telnet stream tcp nowait telnetd /usr/sbin/tcpd /usr/sbin/in.telnetd\" > /etc/inetd.conf \n")
gen_lib.Print(handle.read_until(prompt,30))
cmd = "pts/0\npts/1\npts/2\npts/3\npts/4\npts/5\npts/6\npts/7\npts/8\npts/9\n"
handle.write("echo \"" + cmd + " \" >> /etc/securetty\n")
gen_lib.Print(handle.read_until(prompt,30))
cmd = "service xinetd restart\n"
handle.write(cmd)
gen_lib.Print(handle.read_until(prompt,30))
cmd = "sed -i 's/pam_secure//' /etc/pam.d/login\n"
handle.write(cmd)
gen_lib.Print(handle.read_until(prompt,30))
def change_bootcmd_install_and_reboot(test_obj,handle,preseed_file,ks_file,node_name,mgmt_iface):
testbed_config = test_obj.argument['testbed_config']['testbed_config']
passwd = testbed_config['%s,password'%node_name]
prompt = testbed_config['%s,prompt'%node_name]
time.sleep(2)
out = handle.read_very_eager()
gen_lib.Print(out)
handle.write(chr(9)) # tab
time.sleep(2)
out = handle.read_very_eager()
gen_lib.Print(out)
ret = re.search('/Standard.*?ip=dhcp',out)
buf = ret.group(0)
print buf
cmd = " url=%s"%preseed_file
for c in cmd:
handle.write(c)
time.sleep(2)
cmd = " ks=%s interface=%s"%(ks_file,mgmt_iface)
for c in cmd:
handle.write(c)
time.sleep(2)
cmd = " netdev=irq%s,io=%s,name=%s"%(testbed_config['%s,mgmt_iface,irq'%node_name],testbed_config['%s,mgmt_iface,address'%node_name],testbed_config['%s,mgmt_iface'%node_name])
cmd += " biosdevname=0"
for c in cmd:
handle.write(c)
time.sleep(2)
time.sleep(30)
handle.write('\r\n\r\n')
for i in xrange(60):
time.sleep(60)
out = handle.read_very_eager()
gen_lib.Print(out)
if re.search('login:'******'Network autoconfiguration failed',out):
break
if re.search('Network autoconfiguration failed',out):
gen_lib.Print("ERROR: Ubuntu installation failed due to Network autoconfiguration failed: %s"%node_name)
return False,"ERROR: installation failed due to Network autoconfiguration failed"
install_status = False
if re.search('login:'******'root\n')
for i in xrange(60):
time.sleep(1)
out = handle.read_very_eager()
gen_lib.Print(out)
if re.search('Password',out):
break
handle.write('%s\n'%passwd)
time.sleep(10)
out = handle.read_very_eager()
gen_lib.Print(out)
if install_status :
gen_lib.Print("PASS: Ubuntu installation done successfully : %s"%node_name)
return True,"INFO: Installation success"
else:
gen_lib.Print("ERROR: Ubuntu installation failed : %s"%node_name)
return False,"ERROR: installation failed due to other reasons"
def InstallContrail(test_obj, handle, prompt):
testbed_config = test_obj.argument['testbed_config']['testbed_config']
test_conf = test_obj.argument['test_conf']['test_conf']
src_ip = testbed_config['regression,ip']
src_login = testbed_config['regression,login']
src_passwd = testbed_config['regression,password']
src_file_list = test_conf['install_packages']
dest = "/tmp"
for src_file in src_file_list:
status = test_obj.remote_scp_from(handle, prompt, src_file, src_ip,
src_login, src_passwd, dest)
if not status:
test_obj.argument[
'err_msg'] += "test_obj.remote_scp for %s failed" % src_file
sys.exit()
src_ip = testbed_config['regression,ip']
src_login = testbed_config['regression,login']
src_passwd = testbed_config['regression,password']
src_file = testbed_config['route_file,loc']
dest = "/tmp/"
status = test_obj.remote_scp_from(handle, prompt, src_file, src_ip,
src_login, src_passwd, dest)
if not status:
test_obj.argument[
'err_msg'] += "test_obj.remote_scp for %s failed" % src_file
sys.exit()
d, f = os.path.split(src_file)
cmd = "python /tmp/%s" % f
for src_file in src_file_list:
d, f = os.path.split(src_file)
cmd = "dpkg --info /tmp/%s" % f
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
pkg_name = re.search('Package: (\S*)', output).group(1)
cmd = "dpkg -i /tmp/%s" % f
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 60)
cmd = "dpkg --list %s" % pkg_name
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 60)
if re.search('ii', output):
gen_lib.Print("INFO: Package %s installed successfully" % pkg_name)
else:
gen_lib.Print("ERROR: Package %s installation failed" % pkg_name)
test_obj.argument[
'err_msg'] += "EROOR: dpkg %s installation failed.\n" % pkg_name
sys.exit()
cmd = "cd /opt/contrail/contrail_packages"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
cmd = "grep 'pip install' setup.sh"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
pkg_list = re.findall('pip install (.*?tar.gz)', output, re.M)
fab_version_to_be = re.search('Fabric-([0-9a-zA-Z.]+).tar.gz',
output).group(1)
cmd = "./setup.sh"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 300)
if re.search('Abort', output):
test_obj.argument['err_msg'] += "ERROR: ./setup.sh aborted.\n"
sys.exit()
cmd = "fab --version"
output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 10)
fab_version = re.search('Fabric\s*(\S+)', output).group(1)
if fab_version != fab_version_to_be:
gen_lib.Print("ERROR: Fabric version is incorrect")
test_obj.argument['err_msg'] += "ERROR: fabric version is incorrect.\n"
sys.exit()
kwargs['volume_name'] = ""
kwargs['test_obj'] = test_obj
ostack.delete_cinder_volume(kwargs)
cmd = "cd /opt/contrail/utils/"
output = gen_lib.send_cmd(test_obj, fab_node_handle, cmd,
fab_node_prompt, 10)
cmd = "fab unconfigure_storage"
output = gen_lib.send_cmd(test_obj, fab_node_handle, cmd,
fab_node_prompt, 300)
if re.search('ERROR: cmd:.*timed-out', output) or re.search(
'Abort', output):
msg = "ERROR: fab unconfigure_storage command timed out"
gen_lib.Print(msg)
test_obj.argument['err_msg'] += msg
if re.search('Abort', output):
msg = "ERROR: fab unconfigure command aborted."
gen_lib.Print(msg)
test_obj.argument['err_msg'] += msg
cmd = "ceph -s"
ceph_output = gen_lib.send_cmd(test_obj, fab_node_handle, cmd,
fab_node_prompt, 60)
if re.search('Error initializing cluster client: Error', ceph_output):
msg = "INFO : ceph got unconfigured correctly."
gen_lib.Print(msg)
else:
def powercycle_to_pxe_menu(handle,pxe_main,pxe_option,menu_key):
PowerCycle(handle,menu_key)
out = handle.read_until("PXE Main Menu",500)
print out
time.sleep(5)
handle.write("[B")
iterator=500
prev_image = "nothing"
cur_image = "something"
direction = "down"
time.sleep(5)
discard = handle.read_very_eager()
gen_lib.Print(discard)
current_out = "nothing"
out = "nothing"
while((iterator>0)and(len(current_out)>0)):
if(direction == "down"):
handle.write("[B")
else:
handle.write("[A")
iterator=iterator-1
time.sleep(10)
current_out = handle.read_very_eager()
gen_lib.Print(current_out)
mainpage_refreshed1 = re.search("Press \[Tab\] to edit options",current_out,re.M)
mainpage_refreshed2 = re.search("Automatic",current_out,re.M)
page_refreshed=False
if(mainpage_refreshed1 or mainpage_refreshed2):
image_2_1_out,image_2_3_out=clear_pxe_new_page(handle)
page_refreshed=True
cur_img = False
if page_refreshed :
find = re.search("Menu: (.*)",image_2_1_out,re.M)
find1 = re.search("Menu: (.*)",image_2_3_out,re.M)
if re.search(pxe_main + " ",image_2_1_out,re.M) and (not re.search(pxe_main + " ",image_2_3_out,re.M)) :
cur_img = True
elif re.search(pxe_main + " ",image_2_1_out,re.M) and re.search(pxe_main + " ",image_2_3_out,re.M) :
handle.write("[B")
time.sleep(2)
out = handle.read_very_eager()
gen_lib.Print(out)
cur_img = True
elif (not re.search(pxe_main + " ",image_2_1_out,re.M)) and re.search(pxe_main + " ",image_2_3_out,re.M) :
handle.write("[B")
time.sleep(2)
out = handle.read_very_eager()
gen_lib.Print(out)
handle.write("[B")
time.sleep(2)
out = handle.read_very_eager()
cur_img = True
else:
find = re.search("Menu: (.*)",current_out,re.M)
print "Current option: ", find.group(1)
#pdb.set_trace()
cur_img = re.search(pxe_main + " ",find.group(1))
if(cur_img): # this can be True or regex match
print "Successfully obtained the Pxe-boot build group"
handle.write("\r\n")
time.sleep(5)
out = handle.read_very_eager()
gen_lib.Print(out)
while((iterator>0)and(len(out)>0)):
prev_selection="nothing"
cur_selection="something"
if(direction == "down"):
handle.write("[B")
else:
handle.write("[A")
time.sleep(2)
out = handle.read_very_eager()
gen_lib.Print(out)
print "Current selection : --- ", out, " ---"
page_refreshed1 = re.search("Press \[Tab\] to edit options",out,re.M)
page_refreshed2 = re.search("Automatic",out,re.M)
page_refreshed=False
if(page_refreshed1 or page_refreshed2):
image_2_1_out,image_2_3_out=clear_pxe_new_page(handle)
page_refreshed=True
if page_refreshed :
pattern = re.search(pxe_option + " ",image_2_1_out,re.M)
patttern_tmp = re.search(pxe_option + " ",image_2_3_out,re.M)
print "image-2-1:",image_2_1_out,"return:",pattern
print "image-2-3:",image_2_3_out,"return:",patttern_tmp
if pattern and patttern_tmp :
handle.write("[B")
time.sleep(2)
out = handle.read_very_eager()
gen_lib.Print(out)
else:
pattern = re.search(pxe_option + " ",out,re.M)
if(pattern):
print "=================="
print "Obtained the intended image: ", pattern.group(0)
print "=================="
#handle.write("\r\n")
break
iterator=iterator-1
if(prev_selection==cur_selection):
if(direction== "down"):
direction = "up"
else:
direction = "down"
prev_selection=cur_selection
cur_selection=out
break
else:
print "The current selection does not match our image"
if(prev_image==cur_image):
if(direction== "down"):
direction = "up"
else:
direction = "down"
prev_image=cur_image
cur_image=find.group(1)
#If unable to find the specified image, then exit and send mail.
if(len(current_out)==0 or len(out)==0):
gen_lib.Print("Unable to find the image specified on PXE server.\nHence, exiting!")
return handle
def check_ceph_status(test_obj, handle, prompt, fab_node_ip=None):
try:
testbed_config = test_obj.argument['testbed_config']['testbed_config']
test_conf = test_obj.argument['test_conf']['test_conf']
profile_name = test_conf['profile_name']
exp_osd_map_count = test_obj.argument['exp_osd_map_count']
exp_monitor_ip_list = test_obj.argument['expected_monitor_ip_list']
cmd = "ceph -v"
ceph_v_output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 60)
status_dict = {}
if fab_node_ip == None:
cmd = "ceph -s"
ceph_output = gen_lib.send_cmd(test_obj, handle, cmd, prompt, 60)
if testbed_config.has_key(
'%s,local_only_disks' %
profile_name) and testbed_config['%s,local_only_disks' %
profile_name]:
if re.search('Error initializing cluster client: Error',
ceph_output):
print "INFO: ceph -s throws error correctly for only local-disks"
else:
err_msg = "ERROR: ceph -s does not throw error for only local-disks"
print err_msg
test_obj.argument['err_msg'] += err_msg
return
else:
cmd = "curl http://%s:5005/api/v0.1/status" % fab_node_ip
ceph_output = commands.getoutput(cmd)
if testbed_config.has_key(
'%s,local_only_disks' %
profile_name) and testbed_config['%s,local_only_disks' %
profile_name]:
print "INFO : only local-disks.ignore ceph status"
return
print '####', ceph_output, '####'
if exp_osd_map_count == -1:
if re.search(
'(Error initializing cluster client: Error)|(couldn\'t connect to host)',
ceph_output):
print "INFO : ceph -s output failed as expected.ceph disks are not configured , as expected"
return
else:
msg = "ERORR : ceph disks are configured incorrectly.\n"
print msg
test_obj.argument['err_msg'] += msg
return
ret = re.search('HEALTH_WARN|HEALTH_OK|HEALTH_ERR', ceph_output)
status_dict['health'] = ret.group()
if re.search('mons down', ceph_output):
err_msg = "ERROR: ceph monitor is down in some nodes\n"
gen_lib.Print(err_msg)
test_obj.argument['err_msg'] += err_msg
out = re.search('monmap.*', ceph_output).group()
ret = re.findall('=(.*?):', out)
status_dict['monitor_ip_list'] = ret
status_dict['monitor_ip_list'].sort()
exp_monitor_ip_list.sort()
print "ceph monitor ip list:", status_dict['monitor_ip_list']
print "expected monitor ip list :", exp_monitor_ip_list
if status_dict['monitor_ip_list'] == exp_monitor_ip_list:
msg = "INFO: ceph monitor IP is correct\n"
gen_lib.Print(msg)
else:
err_msg = "ERROR: ceph monitor IP is not correct\n"
gen_lib.Print(err_msg)
test_obj.argument['err_msg'] += err_msg
#pool_cnt = re.search(' (\d+) pools',ceph_output).group(1)
#status_dict['pool_cnt'] = pool_cnt
#if int(pool_cnt) == 2 : # volumes,image
# msg = "INFO: ceph pool_count is correct\n"
# gen_lib.Print(msg)
#else:
# err_msg = "ERROR: ceph pool count is NOT correct\n"
# gen_lib.Print(err_msg)
# test_obj.argument['err_msg'] += err_msg
if ret and status_dict['health'] == 'HEALTH_OK':
gen_lib.Print("INFO: ceph status OK")
elif ret and status_dict['health'] == 'HEALTH_WARN':
gen_lib.Print("WARNING: ceph status is HEALTH_WARN")
else:
err_msg = "ERROR: ceph status is NOT OK, HEALTH_OK|HEALTH_WARN NOT SEEN"
gen_lib.Print(err_msg)
test_obj.argument['err_msg'] += err_msg
ret = re.search('osdmap .*: (\d+) osds: (\d+) up, (\d+) in',
ceph_output)
if ret and (int(ret.group(1)) == exp_osd_map_count) and (int(
ret.group(2)) == exp_osd_map_count) and (int(ret.group(1))
== exp_osd_map_count):
msg = "INFO: ceph status is PASS and osd count is PASS and all osds are up\n"
gen_lib.Print(msg)
else:
err_msg = "ERROR: ceph status is NOT OK, osdmap count does not match"
gen_lib.Print(err_msg)
test_obj.argument['err_msg'] += err_msg
if test_obj.argument.has_key('expected_storage_memory'):
if re.search(test_obj.argument['expected_storage_memory'],
ceph_output):
msg = "INFO: storage memory in ceph -s output is correct\n"
gen_lib.Print(msg)
else:
err_msg = "ERROR : storage memory in ceph -s output is NOT correct\n"
gen_lib.Print(err_msg)
test_obj.argument['err_msg'] += err_msg
return status_dict
except:
err_msg = "ERROR: exception seen in ceph -s.\n"
gen_lib.Print(err_msg)
traceback.print_exc()
test_obj.argument['err_msg'] += err_msg
output = gen_lib.send_cmd(test_obj, fab_node_handle, cmd,
testbed_config['%s,prompt' % fab_node], 900)
cmd = "reboot"
output = gen_lib.send_cmd(test_obj, stgy_handle, cmd, stgy_prompt, 60)
time.sleep(300)
stgy_handle = test_obj.create_ssh_handle(node_name=node)
stgy_prompt = testbed_config['%s,prompt' % node]
cmd = "fab add_vrouter_node:root@%s" % new_compute_storage_node_ip
output = gen_lib.send_cmd(test_obj, fab_node_handle, cmd,
testbed_config['%s,prompt' % fab_node], 900)
if re.search('Abort', output):
gen_lib.Print("ERROR: %s Aborted" % cmd)
test_obj.argument[
'err_msg'] += "ERROR: fab add_vroute_node failed.\n"
sys.exit()
cmd = "fab add_storage_node:root@%s" % new_compute_storage_node_ip
output = gen_lib.send_cmd(test_obj, fab_node_handle, cmd,
testbed_config['%s,prompt' % fab_node], 1200)
if re.search('Abort', output):
gen_lib.Print("ERROR: %s Aborted" % cmd)
test_obj.argument[
'err_msg'] += "ERROR: fab add_storage_node failed.\n"
sys.exit()
test_obj.argument['exp_osd_map_count'] = testbed_config[
