def test_echo_text(self):
stream = PiecewiseStream(u('pi epsilon mu'))
wrapper = streamexpect.wrap(stream, unicode=True, echo=True)
with testfixtures.OutputCapture() as output:
match = wrapper.expect_regex(u('[eu]psilon'))
output.compare('pi epsilon mu')
self.assertTrue(match is not None)
self.assertEqual(u('epsilon'), match.match)
def test_expect_text_twice(self):
stream = PiecewiseStream(u('tau iota mu'), max_chunk=3)
wrapper = streamexpect.wrap(stream, unicode=True)
match = wrapper.expect_text(u('iota'))
self.assertTrue(match is not None)
self.assertEqual(u('iota'), match.match)
match = wrapper.expect_text(u('mu'))
self.assertTrue(match is not None)
self.assertEqual(u('mu'), match.match)
def test_expect_text_twice_with_small_window(self):
stream = PiecewiseStream(u('tau iota epsilon mu'), max_chunk=20)
wrapper = streamexpect.wrap(stream, unicode=True, window=8)
match = wrapper.expect_text(u('iota'))
self.assertTrue(match is not None)
self.assertEqual(u('iota'), match.match)
match = wrapper.expect_text(u('mu'))
self.assertTrue(match is not None)
self.assertEqual(u('mu'), match.match)
def test_expect_bytes(self):
source, drain = socket.socketpair()
try:
wrapper = streamexpect.wrap(drain, unicode=False)
source.sendall(b'tau iota mu')
match = wrapper.expect_bytes(b'iota')
self.assertTrue(match is not None)
self.assertEqual(b'iota', match.match)
finally:
source.close()
drain.close()
def test_expect_bytes_twice_on_split_buffer_with_small_window(self):
source, drain = socket.socketpair()
try:
wrapper = streamexpect.wrap(drain, unicode=False, window=8)
source.sendall(b'tau iota m')
match = wrapper.expect_bytes(b'iota')
self.assertTrue(match is not None)
self.assertEqual(b'iota', match.match)
source.sendall(b'u tau iota')
match = wrapper.expect_bytes(b'mu')
self.assertTrue(match is not None)
self.assertEqual(b'mu', match.match)
finally:
source.close()
drain.close()
def drawbot_init(port):
global ser
global exp
# close, if port was openend before
try:
ser.close()
except:
pass
try:
ser = serial.Serial(port, baudrate=2400, timeout=0)
exp = streamexpect.wrap(ser)
except:
print("Unable to open '{}!'".format(port))
return
ser.write(b"get\r\n")
exp.expect_regex(b"([A-Z]+)\r\n", timeout=TIMEOUT)
def EdgeTxPassthroughEnable(port, requestedBaudrate):
print_header("======== PASSTHROUGH INIT ========")
print_log(" Trying to initialize %s @ %s" % (port, requestedBaudrate))
s = serial.Serial(port=port,
baudrate=requestedBaudrate,
bytesize=8,
parity='N',
stopbits=1,
timeout=1,
xonxoff=0,
rtscts=0)
try:
with streamexpect.wrap(s) as rl:
rl.flush()
rl.write(b"set pulses 0\n")
rl.expect_bytes(b"set: ", timeout=1.0)
rl.expect_bytes(b"> ", timeout=1.0)
rl.write(b"set rfmod 0 power off\n")
rl.expect_bytes(b"set: ", timeout=1.0)
rl.expect_bytes(b"> ", timeout=1.0)
time.sleep(.5)
rl.write(b"set rfmod 0 bootpin 1\n")
rl.expect_bytes(b"set: ", timeout=1.0)
rl.expect_bytes(b"> ", timeout=1.0)
time.sleep(.1)
rl.write(b"set rfmod 0 power on\n")
rl.expect_bytes(b"set: ", timeout=1.0)
rl.expect_bytes(b"> ", timeout=1.0)
time.sleep(.1)
rl.write(b"set rfmod 0 bootpin 0\n")
rl.expect_bytes(b"set: ", timeout=1.0)
rl.expect_bytes(b"> ", timeout=1.0)
cmd = "serialpassthrough rfmod 0 %s" % requestedBaudrate
print_log(" Enabling serial passthrough...")
print_log(" CMD: '%s'" % cmd)
rl.write(cmd.encode("utf-8"))
rl.write(b'\n')
time.sleep(.2)
except streamexpect.ExpectTimeout:
raise SystemExit("[ERROR] PASSTHROUGH init failed!")
s.close()
print_info("======== PASSTHROUGH DONE ========")
def etx_passthrough_init(port, requestedBaudrate):
sys.stdout.flush()
dbg_print("======== PASSTHROUGH INIT ========")
dbg_print(" Trying to initialize %s @ %s" % (port, requestedBaudrate))
s = serial.Serial(port=port,
baudrate=requestedBaudrate,
bytesize=8,
parity='N',
stopbits=1,
timeout=1,
xonxoff=0,
rtscts=0)
with streamexpect.wrap(s) as rl:
rl.flush()
rl.write(b"set pulses 0\n")
rl.expect_bytes(b"set: ", timeout=1.0)
rl.expect_bytes(b"> ", timeout=1.0)
rl.write(b"set rfmod 0 power off\n")
rl.expect_bytes(b"set: ", timeout=1.0)
rl.expect_bytes(b"> ", timeout=1.0)
time.sleep(.5)
rl.write(b"set rfmod 0 bootpin 1\n")
rl.expect_bytes(b"set: ", timeout=1.0)
rl.expect_bytes(b"> ", timeout=1.0)
time.sleep(.1)
rl.write(b"set rfmod 0 power on\n")
rl.expect_bytes(b"set: ", timeout=1.0)
rl.expect_bytes(b"> ", timeout=1.0)
time.sleep(.1)
rl.write(b"set rfmod 0 bootpin 0\n")
rl.expect_bytes(b"set: ", timeout=1.0)
rl.expect_bytes(b"> ", timeout=1.0)
cmd = "serialpassthrough rfmod 0 %s" % requestedBaudrate
dbg_print("Enabling serial passthrough...")
dbg_print(" CMD: '%s'" % cmd)
rl.write(cmd.encode("utf-8"))
rl.write(b'\n')
time.sleep(.2)
s.close()
dbg_print("======== PASSTHROUGH DONE ========")
def start_and_connect_nodes(host_list=None):
"""
Start and connect to other nodes.
It requires controller-0 to be installed already
Args:
host_list(list): list of host names to start and connect to.
Steps:
- Power on nodes and create stream to them.
- Returns sockets and streams for future use
"""
streams = {}
socks = {}
LOG.info(host_list)
port = 10001
for host in host_list:
sock = serial.connect('{}'.format(host), port)
stream = streamexpect.wrap(sock, echo=True, close_stream=False)
time.sleep(10)
socks[host] = sock
streams[host] = stream
port += 1
return socks, streams
def test_unhandled_type(self):
with self.assertRaises(TypeError):
streamexpect.wrap(b'')
def test_context_manager_no_close_stream(self):
stream = EmptyStream()
with streamexpect.wrap(stream, close_stream=False):
pass
self.assertFalse(stream.closed)
def test_context_manager(self):
stream = EmptyStream()
with streamexpect.wrap(stream, close_stream=True):
pass
self.assertTrue(stream.closed)
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (c) 2015, Digi International Inc., All Rights Reserved.
# Send the uname command to a Linux PC connected to a Windows PC over serial
import serial
import streamexpect
# timeout=0 is essential, as streams are required to be non-blocking
ser = serial.Serial('COM1', baudrate=115200, timeout=0)
with streamexpect.wrap(ser) as stream:
stream.write(b'\r\nuname -a\r\n')
match = stream.expect_bytes(b'Linux', timeout=1.0)
print(u'Found Linux at index {}'.format(match.start))
def test_expect_unicode_regex(self):
stream = PiecewiseStream(u('pi epsilon mu'), max_chunk=3)
wrapper = streamexpect.wrap(stream, unicode=True)
match = wrapper.expect_regex(u('[eu]psilon'))
self.assertTrue(match is not None)
self.assertEqual(u('epsilon'), match.match)
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (c) 2015, Digi International Inc., All Rights Reserved.
import io
import streamexpect
unicode_stream = io.StringIO(u'¡Se puede con español!')
with streamexpect.wrap(unicode_stream, unicode=True) as stream:
match = stream.expect_text(u'español')
assert match is not None
print(u'Found {} at index {}'.format(match.match, match.start))
def run_install_scripts(stream, host_list, aio_type=None, storage=False, release='R6', socks=None, streams=None,
labname=None, username='******', password='******', conf_files={}):
"""
Runs lab install.sh iterations. Currently does not support Simplex systems
Args:
stream: Stream to controller-0
host_list: list of hosts, used when running aio scripts to install controller-1 at the appropriate time
release: Release that is installed.
aio_type: Option to run the script for aio setup
storage: Option to run the script for storage setup
streams: Dictionary of streams to nodes
Steps:
- Checks for lab_setup files
- Runs lab_setup iterations
- Unlocks nodes
"""
LOG.info("Starting to run the second round of lab_setup script. ")
if release == 'R6' or release == 'R5' or release == 'R4':
ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/images/ | grep tis-centos-guest.img',
prompt="tis-centos-guest.img", fail_ok=True,
timeout=10)
else:
ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/images/ | grep cgcs-guest.img',
prompt="cgcs-guest.img", fail_ok=True,
timeout=10)
if ret != 0:
LOG.info("Guest image not found. Please transfer the file before continuing. "
"Press enter once guest image is obtained.")
input()
time.sleep(5)
serial.send_bytes(stream, "chmod +x *.sh", timeout=20)
ret = serial.send_bytes(stream, '/bin/ls /home/' + username + '/ | grep lab_setup.sh', prompt="lab_setup.sh",
fail_ok=True, timeout=10)
if ret != 0:
LOG.info("Lab_setup.sh not found. Please transfer the "
"required files before continuing. Press enter once files are obtained.")
input()
conf_str = ""
for file in conf_files:
conf_str = conf_str + " -f {}".format(file)
LOG.info("lab_setup.sh will be run with options: {}".format(conf_str))
start = time.time()
if aio_type:
## FOR AIO
serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone')
if release != 'R6':
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False, fail_ok=True)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
LOG.info("Running system compute-config-complete, "
"installation will resume once controller-0 reboots and services are active")
serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone')
serial.send_bytes(stream, "system compute-config-complete", expect_prompt=False)
serial.expect_bytes(stream, "login:"******"Installing {} ".format(ctrlr1))
host_helper.install_host(stream, ctrlr1, host_type='controller', host_id=2)
# Now wait for controller-1 to come up. Look for "login" on the serial port.
# Close the socket if we are done
try:
serial.expect_bytes(cont1_stream, "ontroller-1 login:"******"{} installation complete".format(ctrlr1))
except Exception as e:
LOG.info("Connection failed for {} with {}.".format(ctrlr1, e))
## Sometimes we get UnicodeDecodeError exception due to the output
## of installation on the serial port, so ignore the exception and try one more time.
if HostTimeout.HOST_INSTALL > (time.time()-start_time):
LOG.info("Ignore the exception and wait for {} to be installed.".format(ctrlr1))
serial.expect_bytes(cont1_stream, "ontroller-1 login:"******"./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging."
" Please re-run the iteration before continuing. Press enter to continue.")
input()
## Unlock controller-1
now = time.time()
ret = host_helper.unlock_host(stream, ctrlr1)
if ret == 1:
LOG.info("Cannot unlock controller-1, pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.")
input()
retry = 0
while retry < 5:
serial.send_bytes(cont1_stream, '\n', expect_prompt=False)
try:
ret = serial.expect_bytes(cont1_stream, "ontroller-1 login:"******"Unlocking controller-1 timed-out. pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.")
input()
break
else:
LOG.info("Unlocking controller-1 time (mins): {}".format((time.time() - now)/60))
if (time.time() - now)/60 < 15.0:
LOG.info("login is found right after host-unlock. Wait and try again.")
now = time.time()
time.sleep(10)
retry += 1
else:
break
except Exception as e:
LOG.info("Unlocking controller-1 failed with {}".format(e))
LOG.info("Pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.")
input()
break
serial.disconnect(socks[labname + '-' + ctrlr1])
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
LOG.info("Completed install successfully.")
else:
## FOR NON-AIO cases
serial.send_bytes(stream, "source /etc/nova/openrc", prompt='keystone')
# TODO (WEI): double check this
# Why only if not R5, we run lab_setup.sh here?
#if release != 'R5':
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
## TODO:
## 1. Unlock sometimes won't bring a node into enabled state. Need to reboot/reset it.
## 2. When unlocking a node, sometimes the node will reboot again after the "login"
## prompt is seen on the serial port, so seeing "login" doesn't mean the node becomes
## "unlocked/enabled". The workaround can be:
## 1) time.sleep(10) and repeat the wait for "login" logic again.
## 2) After seeing "login" prompt, send to controller-0
## "system host-list | grep node" to look for "enabled".
## Unlock controller-1
now = time.time()
ret = host_helper.unlock_host(stream, "controller-1")
if ret == 1:
LOG.info("Cannot unlock controller-1, pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.")
input()
## Wait for controller-1 to come up
retry = 0
ctrlr_1 = labname + "-controller-1"
while retry < 5:
serial.send_bytes(streams[ctrlr_1], '\n', expect_prompt=False)
try:
ret = serial.expect_bytes(streams[ctrlr_1], "ontroller-1 login:"******"Unlocking controller-1 timed-out. pausing to allow for debugging. Press enter to continue.")
input()
break
else:
LOG.info("Unlock controller-1 time (mins): {}".format((time.time() - now)/60))
## Sometimes "login" is found right after "host-unlock" is issued, so we need to wait and try again.
## Also sometimes the node will go thru reboot again after "login" is detected on the serial port.
## As the node unlock time is usually longer than 15 mins, as a temperary workaround
## let's wait at least for 15 mins now.
if (time.time() - now)/60 < 15.0:
LOG.info("Found controller-1 login right away. Need to wait and try again.")
now = time.time()
time.sleep(10)
retry += 1
else:
break
except Exception as e:
LOG.info("Unlock controller-1 failed with {} pausing to allow for debugging. Press enter to continue.".format(e))
input()
break
serial.disconnect(socks[ctrlr_1])
host_list.remove(ctrlr_1)
## TODO: Well, there is a chance that controller-1 is not "unlocked/enabled" yet.
## See comment #2 above.
## If it is a storage lab
if storage:
LOG.info("Re-running lab_setup.sh to configure storage nodes.")
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
now = time.time()
for host in host_list:
host = host[len(labname)+1:]
if host.startswith('storage'):
LOG.info("Unlock {}".format(host))
ret = host_helper.unlock_host(stream, host)
if ret == 1:
LOG.info("Cannot unlock {}, pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.".format(host))
input()
time.sleep(20)
## TODO: Need the retry strategy here too.
for host in host_list:
if 'storage' in host:
serial.send_bytes(streams[host], '\n', expect_prompt=False)
try:
ret = serial.expect_bytes(streams[host], "{} login:"******"Unlocking {} timed-out. Pause for debugging. Press enter to continue.".format(host))
input()
else:
LOG.info("Unlocking {} time (mins): {}".format(host, (time.time() - now)/60))
if (time.time() - now)/60 < 15.0:
LOG.info("Found {} login too soon. Make sure it is up. Press enter to continue.".format(host))
input()
except Exception as e:
LOG.info("Unlock {} failed with {}. Pause for debugging. Press enter to continue.".format(host, e))
input()
serial.disconnect(socks[host])
host_list.remove(host)
LOG.info("Completed storage node unlock")
LOG.info("Re-running lab_setup.sh before unlocking compute nodes.")
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "topping after", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing. Press enter to continue.")
input()
# unlock compute nodes
now = time.time()
for host in host_list:
host = host[len(labname)+1:]
ret = host_helper.unlock_host(stream, host)
if ret == 1:
LOG.info("Cannot unlock {}, pausing to allow for debugging. "
"Please unlock before continuing. Press enter to continue.".format(host))
input()
time.sleep(20)
LOG.info("Waiting for {} to unlock.".format(host_list))
## Check unlocking status
failed_nodes = []
for host in host_list:
# TODO Fix it! 'ogin:' is always found immediately after unlock
# WEI: fixed it by retrying
retry = 0
while retry < 5:
serial.send_bytes(streams[host], '\n', expect_prompt=False)
try:
ret = serial.expect_bytes(streams[host], "{} login:"******"Unlock {} timed-out.".format(host))
failed_nodes.append(host)
break
else:
LOG.info("Unlock {} time (mins): {}".format(host, (time.time() - now)/60))
if (time.time() - now)/60 < 15.0:
LOG.info("login is found right after host-unlock. Wait and try again.")
now = time.time()
time.sleep(10)
retry += 1
else:
break
except Exception as e:
LOG.info("Unlock {} failed with {}".format(host, e))
failed_nodes.append(host)
break
serial.disconnect(socks[host])
## Let's reset the VMs that failed to unlock
if failed_nodes:
vboxmanage.vboxmanage_controlvms(failed_nodes, action="reset")
time.sleep(10)
tmp_streams = {}
tmp_socks = {}
LOG.info(failed_nodes)
port = 10001
for host in failed_nodes:
tmp_sock = serial.connect('{}'.format(host), port)
tmp_stream = streamexpect.wrap(tmp_sock, echo=True, close_stream=False)
time.sleep(10)
tmp_socks[host] = tmp_sock
tmp_streams[host] = tmp_stream
port += 1
host_failed = False
for host in failed_nodes:
serial.send_bytes(tmp_streams[host], '\n', expect_prompt=False)
try:
ret = serial.expect_bytes(tmp_streams[host], "{} login:"******"{} timed-out to become unlocked/available after reset.".format(host))
host_failed = True
else:
LOG.info("{} became unlocked/available after reset. time (mins): {}".format(host, (time.time() - now)/60))
except Exception as e:
LOG.info("{} failed to become unlocked/available after reset with {}".format(host, e))
host_failed = True
serial.disconnect(tmp_socks[host])
if host_failed:
LOG.info("Not all the nodes are unlocked successfully. Pausing to allow for debugging. "
"Once they all become unlocked/enabled/available, press enter to continue.")
input()
serial.send_bytes(stream, "./lab_setup.sh {}".format(conf_str), expect_prompt=False)
host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "Done", timeout=HostTimeout.LAB_INSTALL, fail_ok=True)
if ret != 0:
LOG.info("Lab_setup.sh failed. Pausing to allow for debugging. "
"Please re-run the iteration before continuing."
" Press enter to continue.")
input()
LOG.info("Completed lab install.")
kpi.LABTIME = time.time()-start
LOG.info("Lab install time: {}".format(kpi.LABTIME/60))
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (c) 2015, Digi International Inc., All Rights Reserved.
import socket
import streamexpect
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('www.google.com', 80))
# By default, this will open in binary mode. To read a non-ASCII text stream,
# the unicode option needs to be enabled.
with streamexpect.wrap(sock) as stream:
# Send the request. This is passed to the underlying socket.
stream.sendall(b'GET / HTTP/1.1\r\nHost: www.google.com\r\n\r\n')
# Find the start of the response.
stream.expect_bytes(b'HTTP/1.1 200 OK\r\n')
# Find the "Date" header using regex groups and print it.
match = stream.expect_regex(br'Date: ([^\r\n]+)\r\n')
print(u'Google says the date/time is ' + match.groups[0].decode('ascii'))
if vboxoptions.createlab:
controller_type = create_lab(vboxoptions)
## WZWZ to debug
if not vboxoptions.debug_rest:
vboxmanage.vboxmanage_startvm(ctrlr0)
else:
time.sleep(30)
node_list = get_all_vms(vboxoptions.labname, option="vms")
LOG.info(node_list)
assert ctrlr0 in node_list, "controller-0 not in vm list. Stopping installation."
sock = serial.connect(ctrlr0, 10000)
cont0_stream = streamexpect.wrap(sock, echo=True, close_stream=False)
try:
if not vboxoptions.debug_rest:
if vboxoptions.createlab:
install_controller_0(cont0_stream, controller_type, vboxoptions.securityprofile,
vboxoptions.release, vboxoptions.lowlatency,
install_mode=vboxoptions.install_mode, ctrlr0_ip=vboxoptions.controller0_ip,
hostadapter_ip=vboxoptions.hostadapter_ip,
username=vboxoptions.username, password=vboxoptions.password)
else:
host_helper.login(cont0_stream, timeout=60, username=vboxoptions.username, password=vboxoptions.password)
setup_networking(cont0_stream, vboxoptions.release, vboxoptions.controller0_ip,
hostadapter_ip=vboxoptions.hostadapter_ip, password=vboxoptions.password)
## WZWZ DEBUG
