def test07_reboot_and_check(self):
print "**** Reboot and check installation."
# create startup files
self.assertEqual(
fit_common.remote_shell(
"touch /etc/default/on-dhcp-proxy /etc/default/on-http /etc/default/on-tftp /etc/default/on-syslog /etc/default/on-taskgraph"
)['exitcode'], 0, "Install failure.")
# reboot
print "**** Rebooting appliance..."
fit_common.remote_shell("shutdown -r now")
print "**** Waiting for login..."
fit_common.countdown(30)
shell_data = 0
for dummy in range(0, 30):
shell_data = fit_common.remote_shell("pwd")
if shell_data['exitcode'] == 0:
break
else:
fit_common.time.sleep(5)
self.assertEqual(shell_data['exitcode'], 0,
"Shell test failed after appliance reboot")
fit_common.time.sleep(10)
self.assertEqual(
fit_common.rackhdapi("/api/2.0/config")['status'], 200,
"Unable to contact RackHD.")
def test00_update_config(self):
# this will add proxy settings to default OnRack Config file
monorail_config = fit_common.rackhdapi('/api/2.0/config')['json']
monorail_config.update({
"httpProxies": [{
"localPath":
"/mirror",
"remotePath":
"/",
"server":
fit_common.GLOBAL_CONFIG['repos']['mirror']
}]
})
monorail_json = open('monorail.json', 'w')
monorail_json.write(
fit_common.json.dumps(monorail_config, sort_keys=True, indent=4))
monorail_json.close()
fit_common.scp_file_to_ora('monorail.json')
self.assertEqual(
fit_common.remote_shell('cp monorail.json /opt/onrack/etc/')
['exitcode'], 0, "RackHD Config file failure.")
os.remove('monorail.json')
print "**** Restart services..."
fit_common.remote_shell("/opt/onrack/bin/monorail restart")
fit_common.countdown(30)
self.assertEqual(
fit_common.rackhdapi("/api/2.0/config")['status'], 200,
"Unable to contact Onrack.")
def test04_power_on_nodes(self):
# BareMetal test bed uses a power strip that we can control via telnet
# Need to add calls to that utility, for now, skipping power on
baremetal_testbed = True
if baremetal_testbed:
log.info(" *** CDU power control not implemented yet, assuming nodes powered on outside this script")
log.info(" *** Sleeping 90 seconds, allow physical nodes to power on")
# Allow physical nodes some time to power on
time.sleep(90)
return
# This powers on nodes via PDU or, if no PDU, power cycles nodes via IPMI to start discovery
# ServerTech PDU case
if pdu_lib.check_pdu_type() != "Unknown":
log.info_5('**** PDU found, powering on PDU outlets')
self.assertTrue(pdu_lib.pdu_control_compute_nodes("on"), 'Failed to power on all outlets')
# Wait about 30 seconds for the outlets to all come on and nodes to DHCP
fit_common.countdown(30)
# no PDU case
else:
log.info_5('**** No supported PDU found, restarting nodes using IPMI.')
# Check if some nodes are already discovered
nodes = []
nodes = fit_common.node_select()
if nodes:
log.info_5(" Nodes already discovered, skipping ipmi power reset")
log.info_5(" %s", nodes)
else:
# Power cycle all nodes via IPMI, display warning if no nodes found
if fit_common.power_control_all_nodes("off") == 0:
log.info_5('**** No BMC IP addresses found in arp table, continuing without node restart.')
else:
# power on all nodes under any circumstances
fit_common.power_control_all_nodes("on")
def test02_post_install_reboot(self):
print "**** Rebooting appliance."
shell_data = fit_common.remote_shell("shutdown -r now")
self.assertEqual(shell_data['exitcode'], 0, 'ORA reboot registered error')
fit_common.countdown(90)
print "**** Waiting for login."
for dummy in range(0, 30):
shell_data = fit_common.remote_shell("pwd")
if shell_data['exitcode'] == 0:
break
else:
fit_common.time.sleep(10)
self.assertEqual(shell_data['exitcode'], 0, "Shell test failed after appliance reboot")
def test02_power_on_nodes(self):
# ServerTech PDU case
if pdu_lib.check_pdu_type() != "Unknown":
print '**** PDU found, powering on PDU outlets'
self.assertTrue(pdu_lib.pdu_control_compute_nodes("on"), 'Failed to power on all outlets')
# Wait about 30 seconds for the outlets to all come on and nodes to DHCP
fit_common.countdown(30)
# no PDU case
else:
print '**** No supported PDU found, restarting nodes using IMPI.'
# Power off all nodes
self.assertNotEqual(fit_common.power_control_all_nodes("off"), 0, 'No BMC IP addresses found')
# Power on all nodes
self.assertNotEqual(fit_common.power_control_all_nodes("on"), 0, 'No BMC IP addresses found')
def test02_power_on_nodes(self):
# ServerTech PDU case
if pdu_lib.check_pdu_type() != "Unknown":
print '**** PDU found, powering on PDU outlets'
self.assertTrue(pdu_lib.pdu_control_compute_nodes("on"),
'Failed to power on all outlets')
# Wait about 30 seconds for the outlets to all come on and nodes to DHCP
fit_common.countdown(30)
# no PDU case
else:
print '**** No supported PDU found, restarting nodes using IMPI.'
# Power off all nodes
self.assertNotEqual(fit_common.power_control_all_nodes("off"), 0,
'No BMC IP addresses found')
# Power on all nodes
self.assertNotEqual(fit_common.power_control_all_nodes("on"), 0,
'No BMC IP addresses found')
def test04_power_on_nodes(self):
# This powers on nodes via PDU or, if no PDU, power cycles nodes via IPMI to start discovery
# ServerTech PDU case
if pdu_lib.check_pdu_type() != "Unknown":
log.info_5('**** PDU found, powering on PDU outlets')
self.assertTrue(pdu_lib.pdu_control_compute_nodes("on"), 'Failed to power on all outlets')
# Wait about 30 seconds for the outlets to all come on and nodes to DHCP
fit_common.countdown(30)
# no PDU case
else:
log.info_5('**** No supported PDU found, restarting nodes using IPMI.')
# Power cycle all nodes via IPMI, display warning if no nodes found
if fit_common.power_control_all_nodes("off") == 0:
log.info_5('**** No BMC IP addresses found in arp table, continuing without node restart.')
else:
# power on all nodes under any circumstances
fit_common.power_control_all_nodes("on")
def test04_power_on_nodes(self):
# This powers on nodes via PDU or, if no PDU, power cycles nodes via IPMI to start discovery
# ServerTech PDU case
if pdu_lib.check_pdu_type() != "Unknown":
log.info_5('**** PDU found, powering on PDU outlets')
self.assertTrue(pdu_lib.pdu_control_compute_nodes("on"), 'Failed to power on all outlets')
# Wait about 30 seconds for the outlets to all come on and nodes to DHCP
fit_common.countdown(30)
# no PDU case
else:
log.info_5('**** No supported PDU found, restarting nodes using IPMI.')
# Power cycle all nodes via IPMI, display warning if no nodes found
if fit_common.power_control_all_nodes("off") == 0:
log.info_5('**** No BMC IP addresses found in arp table, continuing without node restart.')
else:
# power on all nodes under any circumstances
fit_common.power_control_all_nodes("on")
def test05_reboot_and_check(self):
print "**** Reboot and check installation."
self.assertEqual(fit_common.remote_shell(
"touch /etc/default/on-dhcp-proxy /etc/default/on-http /etc/default/on-tftp /etc/default/on-syslog /etc/default/on-taskgraph"
)['exitcode'], 0, "Install failure.")
# reboot
self.assertEqual(fit_common.remote_shell("reboot")['exitcode'], 0, 'ORA reboot registered error')
print "**** Rebooting appliance..."
fit_common.countdown(30)
print "**** Waiting for login..."
shell_data = 0
for dummy in range(0, 30):
shell_data = fit_common.remote_shell("pwd")
if shell_data['exitcode'] == 0:
break
else:
fit_common.time.sleep(5)
self.assertEqual(shell_data['exitcode'], 0, "Shell test failed after appliance reboot")
fit_common.time.sleep(10)
self.assertEqual(fit_common.rackhdapi("/api/1.1/config")['status'], 200, "Unable to contact RackHD.")
def deploy_ova(self, vm, uname, passwd, numvms, ovafile):
print '**** Deploying OVA file on hypervisor ' + fit_common.fitargs(
)['hyper']
rc = subprocess.call(
"ovftool --X:injectOvfEnv --overwrite "
"--powerOffTarget --skipManifestCheck -q "
"--net:'ADMIN'='VM Network' "
"--net:'CONTROL'='Control Network' "
"--net:'PDU'='PDU Network' "
"--name='ora-stack-" + fit_common.fitargs()['stack'] + "-" +
str(vm) + "' "
"--noSSLVerify " + ovafile + " vi://" + uname + ":" + passwd +
"@" + fit_common.fitargs()['hyper'],
shell=True)
# Check for successful completion
if rc > 0:
print 'OVA installer failed at host: ' + fit_common.fitargs(
)['hyper'] + "Exiting..."
sys.exit(255)
# Wait for VM to settle
fit_common.countdown(30)
# check number of vms for deployment
if numvms == 1:
ovamac = fit_common.fitcfg()['ovamac']
else:
# compose appropriate MAC address using lab convention
vmnum = '{0:02}'.format(int(vm))
macsplit = fit_common.fitcfg()['ovamac'].split(":")
ovamac = macsplit[0] + ":" + macsplit[1] + ":" + macsplit[
2] + ":" + macsplit[3] + ":" + macsplit[4] + ":" + vmnum
# Install MAC address by editing OVA .vmx file, then startup VM
esxi_command = "export fullpath=`find vmfs -name ora-stack-" + fit_common.fitargs()['stack'] + "-" + str(vm) + "*.vmx`;" \
"for file in $fullpath;" \
"do " \
"export editline=`cat $file |grep \\\'ethernet0.generatedAddress =\\\'`;" \
"export editcmd=\\\'/\\\'$editline\\\'\/ c\\\ethernet0.address = \\\"" + ovamac + "\\\"\\\';" \
"sed -i \\\"$editcmd\\\" $file;" \
"sed -i \\\'/ethernet0.addressType = \\\"vpx\\\"/ c\\\ethernet0.addressType = \\\"static\\\"\\\' $file;" \
"sed -i \\\'/ethernet0.addressType = \\\"generated\\\"/ c\\\ethernet0.addressType = \\\"static\\\"\\\' $file;" \
"done;" \
"sleep 5;" \
"export vmidstring=`vim-cmd vmsvc/getallvms |grep ora-stack-" + fit_common.fitargs()['stack'] + "-" + str(vm) + "`;" \
"for vmid in $vmidstring;" \
"do " \
"vim-cmd vmsvc/power.on $vmid;" \
"exit $?;" \
"done;"
(command_output, exitstatus) = \
fit_common.pexpect.run(
"ssh -q -o StrictHostKeyChecking=no -t " + uname + "@"
+ fit_common.fitargs()['hyper'] + " " + esxi_command,
withexitstatus=1,
events={"assword": passwd + "\n"},
timeout=20, logfile=sys.stdout)
if exitstatus > 0:
print "MAC address processing failed. Exiting..."
sys.exit(255)
# Poll the OVA via ping
for dummy in range(0, 30):
if vm > 0:
hostname = "stack" + fit_common.fitargs(
)['stack'] + "-ora-" + str(vm) + ".admin"
else:
hostname = "stack" + fit_common.fitargs(
)['stack'] + "-ora.admin"
rc = subprocess.call("ping -c 1 -w 5 " + hostname, shell=True)
if rc == 0:
break
else:
fit_common.time.sleep(10)
self.assertEqual(rc, 0, "VM did not activate.")
# Sync time on ORA
localdate = fit_common.subprocess.check_output("date +%s", shell=True)
fit_common.remote_shell("/bin/date -s @" +
localdate.replace("\n", "") +
";/sbin/hwclock --systohc")
return None
def test01_install_ova_template(self):
ovafile = fit_common.ARGS_LIST['template']
# Check for ovftool
self.assertEqual(
fit_common.subprocess.call('which ovftool', shell=True), 0,
"FAILURE: 'ovftool' not installed.")
# Ping for valid ESXi host
self.assertEqual(
fit_common.subprocess.call('ping -c 1 ' +
fit_common.ARGS_LIST['hyper'],
shell=True), 0,
"FAILURE: ESXi hypervisor not found.")
# Shutdown previous ORA
if fit_common.subprocess.call('ping -c 1 ' +
fit_common.ARGS_LIST['ora'],
shell=True) == 0:
fit_common.remote_shell('shutdown -h now')
fit_common.time.sleep(5)
# Run probe to check for valid OVA file
rc = fit_common.subprocess.call("ovftool " + ovafile, shell=True)
self.assertEqual(rc, 0, 'Invalid or missing OVA file: ' + ovafile)
# this clears the hypervisor ssh key from ~/.ssh/known_hosts
subprocess.call([
"touch ~/.ssh/known_hosts;ssh-keygen -R " +
fit_common.ARGS_LIST['hyper'] +
" -f ~/.ssh/known_hosts >/dev/null 2>&1"
],
shell=True)
# Find correct hypervisor credentials by testing each entry in the list
cred_list = fit_common.GLOBAL_CONFIG['credentials']['hyper']
for entry in cred_list:
uname = entry['username']
passwd = entry['password']
(command_output, exitstatus) = \
fit_common.pexpect.run(
"ssh -q -o StrictHostKeyChecking=no -t " + uname + "@"
+ fit_common.ARGS_LIST['hyper'] + " pwd",
withexitstatus=1,
events={"assword": passwd + "\n"},
timeout=20, logfile=None)
if exitstatus == 0:
break
# Run OVA installer
print '**** Deploying OVA file on hypervisor ' + fit_common.ARGS_LIST[
'hyper']
rc = subprocess.call(
"ovftool --X:injectOvfEnv --overwrite "
"--powerOffTarget --skipManifestCheck -q "
"--net:'ADMIN'='VM Network' "
"--net:'CONTROL'='Control Network' "
"--net:'PDU'='PDU Network' "
"--name='ora-stack-" + fit_common.ARGS_LIST['stack'] + "' "
"--noSSLVerify " + ovafile + " vi://" + uname + ":" + passwd +
"@" + fit_common.ARGS_LIST['hyper'],
shell=True)
# Check for successful completion
if rc > 0:
print 'OVA installer failed at host: ' + fit_common.ARGS_LIST[
'hyper'] + "Exiting..."
sys.exit(255)
# Wait for VM to settle
fit_common.countdown(30)
# Install MAC address by editing OVA .vmx file, then startup VM
esxi_command = "export fullpath=`find vmfs -name ora-stack-" + fit_common.ARGS_LIST['stack'] + "*.vmx`;" \
"for file in $fullpath;" \
"do " \
"export editline=`cat $file |grep \\\'ethernet0.generatedAddress =\\\'`;" \
"export editcmd=\\\'/\\\'$editline\\\'\/ c\\\ethernet0.address = \\\"" + fit_common.STACK_CONFIG[fit_common.ARGS_LIST['stack']]['ovamac'] + "\\\"\\\';" \
"sed -i \\\"$editcmd\\\" $file;" \
"sed -i \\\'/ethernet0.addressType = \\\"vpx\\\"/ c\\\ethernet0.addressType = \\\"static\\\"\\\' $file;" \
"sed -i \\\'/ethernet0.addressType = \\\"generated\\\"/ c\\\ethernet0.addressType = \\\"static\\\"\\\' $file;" \
"done;" \
"sleep 5;" \
"export vmidstring=`vim-cmd vmsvc/getallvms |grep ora-stack-" + fit_common.ARGS_LIST['stack'] + "`;" \
"for vmid in $vmidstring;" \
"do " \
"vim-cmd vmsvc/power.on $vmid;" \
"exit $?;" \
"done;"
(command_output, exitstatus) = \
fit_common.pexpect.run(
"ssh -q -o StrictHostKeyChecking=no -t " + uname + "@"
+ fit_common.ARGS_LIST['hyper'] + " " + esxi_command,
withexitstatus=1,
events={"assword": passwd + "\n"},
timeout=20, logfile=sys.stdout)
if exitstatus > 0:
print "MAC address processing failed. Exiting..."
sys.exit(255)
# Poll the OVA via ping
for dummy in range(0, 30):
rc = subprocess.call("ping -c 1 -w 5 " +
fit_common.ARGS_LIST['ora'],
shell=True)
if rc == 0:
break
else:
fit_common.time.sleep(10)
self.assertEqual(rc, 0, "VM did not activate.")
# Sync time on ORA
localdate = fit_common.subprocess.check_output("date +%s", shell=True)
fit_common.remote_shell("/bin/date -s @" +
localdate.replace("\n", "") +
";/sbin/hwclock --systohc")
def test01_install_ova_template(self):
ovafile = fit_common.ARGS_LIST['template']
# Check for ovftool
self.assertEqual(fit_common.subprocess.call('which ovftool', shell=True), 0, "FAILURE: 'ovftool' not installed.")
# Ping for valid ESXi host
self.assertEqual(fit_common.subprocess.call('ping -c 1 ' + fit_common.ARGS_LIST['hyper'], shell=True), 0, "FAILURE: ESXi hypervisor not found.")
# Shutdown previous ORA
if fit_common.subprocess.call('ping -c 1 ' + fit_common.ARGS_LIST['ora'], shell=True) == 0:
fit_common.remote_shell('shutdown -h now')
fit_common.time.sleep(5)
# Run probe to check for valid OVA file
rc = fit_common.subprocess.call("ovftool " + ovafile, shell=True)
self.assertEqual(rc, 0,'Invalid or missing OVA file: ' + ovafile)
# this clears the hypervisor ssh key from ~/.ssh/known_hosts
subprocess.call(["touch ~/.ssh/known_hosts;ssh-keygen -R "
+ fit_common.ARGS_LIST['hyper'] + " -f ~/.ssh/known_hosts >/dev/null 2>&1"], shell=True)
# Find correct hypervisor credentials by testing each entry in the list
cred_list = fit_common.GLOBAL_CONFIG['credentials']['hyper']
for entry in cred_list:
uname = entry['username']
passwd = entry['password']
(command_output, exitstatus) = \
fit_common.pexpect.run(
"ssh -q -o StrictHostKeyChecking=no -t " + uname + "@"
+ fit_common.ARGS_LIST['hyper'] + " pwd",
withexitstatus=1,
events={"assword": passwd + "\n"},
timeout=20, logfile=None)
if exitstatus == 0:
break
# Run OVA installer
print '**** Deploying OVA file on hypervisor ' + fit_common.ARGS_LIST['hyper']
rc = subprocess.call("ovftool --X:injectOvfEnv --overwrite "
"--powerOffTarget --skipManifestCheck -q "
"--net:'ADMIN'='VM Network' "
"--net:'CONTROL'='Control Network' "
"--net:'PDU'='PDU Network' "
"--name='ora-stack-" + fit_common.ARGS_LIST['stack'] + "' "
"--noSSLVerify "
+ ovafile
+ " vi://" + uname + ":" + passwd + "@"
+ fit_common.ARGS_LIST['hyper'], shell=True)
# Check for successful completion
if rc > 0:
print 'OVA installer failed at host: ' + fit_common.ARGS_LIST['hyper'] + "Exiting..."
sys.exit(255)
# Wait for VM to settle
fit_common.countdown(30)
# Install MAC address by editing OVA .vmx file, then startup VM
esxi_command = "export fullpath=`find vmfs -name ora-stack-" + fit_common.ARGS_LIST['stack'] + "*.vmx`;" \
"for file in $fullpath;" \
"do " \
"export editline=`cat $file |grep \\\'ethernet0.generatedAddress =\\\'`;" \
"export editcmd=\\\'/\\\'$editline\\\'\/ c\\\ethernet0.address = \\\"" + fit_common.STACK_CONFIG[fit_common.ARGS_LIST['stack']]['ovamac'] + "\\\"\\\';" \
"sed -i \\\"$editcmd\\\" $file;" \
"sed -i \\\'/ethernet0.addressType = \\\"vpx\\\"/ c\\\ethernet0.addressType = \\\"static\\\"\\\' $file;" \
"sed -i \\\'/ethernet0.addressType = \\\"generated\\\"/ c\\\ethernet0.addressType = \\\"static\\\"\\\' $file;" \
"done;" \
"sleep 5;" \
"export vmidstring=`vim-cmd vmsvc/getallvms |grep ora-stack-" + fit_common.ARGS_LIST['stack'] + "`;" \
"for vmid in $vmidstring;" \
"do " \
"vim-cmd vmsvc/power.on $vmid;" \
"exit $?;" \
"done;"
(command_output, exitstatus) = \
fit_common.pexpect.run(
"ssh -q -o StrictHostKeyChecking=no -t " + uname + "@"
+ fit_common.ARGS_LIST['hyper'] + " " + esxi_command,
withexitstatus=1,
events={"assword": passwd + "\n"},
timeout=20, logfile=sys.stdout)
if exitstatus > 0:
print "MAC address processing failed. Exiting..."
sys.exit(255)
# Poll the OVA via ping
for dummy in range(0, 30):
rc = subprocess.call("ping -c 1 -w 5 " + fit_common.ARGS_LIST['ora'], shell=True)
if rc == 0:
break
else:
fit_common.time.sleep(10)
self.assertEqual(rc, 0, "VM did not activate.")
# Sync time on ORA
localdate = fit_common.subprocess.check_output("date +%s", shell=True)
fit_common.remote_shell("/bin/date -s @" + localdate.replace("\n", "") + ";/sbin/hwclock --systohc")
def deploy_ova(self,vm,uname,passwd,numvms,ovafile):
print '**** Deploying OVA file on hypervisor ' + fit_common.fitargs()['hyper']
rc = subprocess.call("ovftool --X:injectOvfEnv --overwrite "
"--powerOffTarget --skipManifestCheck -q "
"--net:'ADMIN'='VM Network' "
"--net:'CONTROL'='Control Network' "
"--net:'PDU'='PDU Network' "
"--name='ora-stack-" + fit_common.fitargs()['stack'] + "-" + str(vm) + "' "
"--noSSLVerify "
+ ovafile
+ " vi://" + uname + ":" + passwd + "@"
+ fit_common.fitargs()['hyper'], shell=True)
# Check for successful completion
if rc > 0:
print 'OVA installer failed at host: ' + fit_common.fitargs()['hyper'] + "Exiting..."
sys.exit(255)
# Wait for VM to settle
fit_common.countdown(30)
# check number of vms for deployment
if numvms == 1:
ovamac = fit_common.fitcfg()['ovamac']
else:
# compose appropriate MAC address using lab convention
vmnum = '{0:02}'.format(int(vm))
macsplit = fit_common.fitcfg()['ovamac'].split(":")
ovamac = macsplit[0] + ":" + macsplit[1] + ":" + macsplit[2] + ":" + macsplit[3] + ":" + macsplit[4] + ":" + vmnum
# Install MAC address by editing OVA .vmx file, then startup VM
esxi_command = "export fullpath=`find vmfs -name ora-stack-" + fit_common.fitargs()['stack'] + "-" + str(vm) + "*.vmx`;" \
"for file in $fullpath;" \
"do " \
"export editline=`cat $file |grep \\\'ethernet0.generatedAddress =\\\'`;" \
"export editcmd=\\\'/\\\'$editline\\\'\/ c\\\ethernet0.address = \\\"" + ovamac + "\\\"\\\';" \
"sed -i \\\"$editcmd\\\" $file;" \
"sed -i \\\'/ethernet0.addressType = \\\"vpx\\\"/ c\\\ethernet0.addressType = \\\"static\\\"\\\' $file;" \
"sed -i \\\'/ethernet0.addressType = \\\"generated\\\"/ c\\\ethernet0.addressType = \\\"static\\\"\\\' $file;" \
"done;" \
"sleep 5;" \
"export vmidstring=`vim-cmd vmsvc/getallvms |grep ora-stack-" + fit_common.fitargs()['stack'] + "-" + str(vm) + "`;" \
"for vmid in $vmidstring;" \
"do " \
"vim-cmd vmsvc/power.on $vmid;" \
"exit $?;" \
"done;"
(command_output, exitstatus) = \
fit_common.pexpect.run(
"ssh -q -o StrictHostKeyChecking=no -t " + uname + "@"
+ fit_common.fitargs()['hyper'] + " " + esxi_command,
withexitstatus=1,
events={"assword": passwd + "\n"},
timeout=20, logfile=sys.stdout)
if exitstatus > 0:
print "MAC address processing failed. Exiting..."
sys.exit(255)
# Poll the OVA via ping
for dummy in range(0, 30):
if vm > 0:
hostname = "stack" + fit_common.fitargs()['stack'] + "-ora-" + str(vm) + ".admin"
else:
hostname = "stack" + fit_common.fitargs()['stack'] + "-ora.admin"
rc = subprocess.call("ping -c 1 -w 5 " + hostname, shell=True)
if rc == 0:
break
else:
fit_common.time.sleep(10)
self.assertEqual(rc, 0, "VM did not activate.")
# Sync time on ORA
localdate = fit_common.subprocess.check_output("date +%s", shell=True)
fit_common.remote_shell("/bin/date -s @" + localdate.replace("\n", "") + ";/sbin/hwclock --systohc")
return None