def run_test(build, machine, test, path):
# Open files for raw output from the victim and log data from the test
raw_file_name = os.path.join(path, RAW_FILE_NAME)
debug.verbose('open %s for raw output' % raw_file_name)
raw_file = open(raw_file_name, 'w')
# run the test, dumping the output to the raw file as we go
try:
debug.verbose('harness: setup test')
test.setup(build, machine, path)
debug.verbose('harness: run test')
starttime = datetime.datetime.now()
for out in test.run(build, machine, path):
# timedelta for the time this line was emitted from the start of the run
timestamp = datetime.datetime.now() - starttime
# format as string, discarding sub-second precision
timestr = str(timestamp).split('.', 1)[0]
# filter output line of control characters
filtered_out = filter(lambda c: c in string.printable, out.rstrip())
# debug filtered output along with timestamp
debug.debug('[%s] %s' % (timestr, filtered_out))
# log full raw line (without timestamp) to output file
raw_file.write(out)
debug.verbose('harness: output complete')
except KeyboardInterrupt:
# let the user know that we are on our way out
debug.error('Interrupted! Performing cleanup...')
raise
finally:
raw_file.close()
debug.verbose('harness: cleanup test')
test.cleanup(machine)
def _run_ipbench(self, args, logfile):
cmd = [siteconfig.get('IPBENCH_PATH')] + args
firstrun = True
for _ in range(IPBENCH_ITERATIONS):
if firstrun:
firstrun = False
else:
# sleep a moment to let things settle down between runs
debug.verbose('sleeping between ipbench runs')
time.sleep(IPBENCH_SLEEPTIME)
debug.verbose('running ipbench: %s' % ' '.join(cmd))
child = subprocess.Popen(cmd, stdout=subprocess.PIPE)
timeout = datetime.datetime.now() + IPBENCH_TIMEOUT
while True:
# wait for some output
(rlist, _, _) = select_timeout(timeout, [child.stdout])
if not rlist:
debug.warning('ipbench run timed out')
child.terminate()
child.wait()
raise TimeoutError('waiting for ipbench')
# read one char at a time to avoid blocking
c = child.stdout.read(1)
if c == '':
break # EOF
logfile.write(c)
child.wait()
assert(child.returncode == 0) # check for successful exit
def lock(self):
"""Use conserver to lock the machine."""
# find out current status of console
debug.verbose('executing "console -i %s" to check state' %
self.get_machine_name())
proc = subprocess.Popen(["console", "-i", self.get_machine_name()],
stdout=subprocess.PIPE)
line = proc.communicate()[0]
assert(proc.returncode == 0)
# check that nobody else has it open for writing
myuser = getpass.getuser()
parts = line.strip().split(':')
conname, child, contype, details, users, state = parts[:6]
if users:
for userinfo in users.split(','):
mode, username, host, port = userinfo.split('@')[:4]
if 'w' in mode and username != myuser:
raise MachineLockedError # Machine is not free
# run a console in the background to 'hold' the lock and read output
debug.verbose('starting "console %s"' % self.get_machine_name())
# run on a PTY to work around terminal mangling code in console
(self.masterfd, slavefd) = pty.openpty()
self.lockprocess = subprocess.Popen(["console", self.get_machine_name()],
close_fds=True,
stdout=slavefd, stdin=slavefd)
os.close(slavefd)
# XXX: open in binary mode with no buffering
# otherwise select.select() may block when there is data in the buffer
self.console_out = os.fdopen(self.masterfd, 'rb', 0)
def _runtest(self, target, nclients, nconns, rate):
self.nruns += 1
nrun = self.nruns
httperfs = []
try:
for nclient in range(nclients):
user, host = siteconfig.site.get_load_generator()
assert(nrun < 100 and nclient < 100)
filename = 'httperf_run%02d_%02d.txt' % (nrun, nclient)
logfile = open(os.path.join(self.testdir, filename), 'w')
debug.verbose('spawning httperf on %s' % host)
hp = HTTPerfClient(logfile, user, host, target, nconns, rate)
httperfs.append(hp)
# loop collecting output from all of them
busy_httperfs = list(httperfs) # copy list
timeout = datetime.datetime.now() + HTTPERF_TIMEOUT
while busy_httperfs:
(ready, _, _) = select_timeout(timeout, busy_httperfs)
if not ready:
raise TimeoutError('waiting for httperfs')
for hp in ready:
try:
hp.read()
except EOFError:
busy_httperfs.remove(hp)
finally:
debug.log('cleaning up httperf test...')
for hp in httperfs:
hp.cleanup()
def reboot(self):
self._kill_child()
cmd = self._get_cmdline()
debug.verbose('starting "%s"' % ' '.join(cmd))
devnull = open(os.devnull, 'r')
self.child = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=devnull)
devnull.close()
def unlock(self):
if self.lockprocess is None:
return # noop
debug.verbose("terminating console process (%d)" % self.lockprocess.pid)
os.kill(self.lockprocess.pid, signal.SIGTERM)
self.lockprocess.wait()
self.lockprocess = None
def reboot(self):
self._kill_child()
cmd = self._get_cmdline()
debug.verbose('starting "%s" in gem5.py:reboot' % ' '.join(cmd))
devnull = open('/dev/null', 'w')
self.child = subprocess.Popen(cmd, stderr=devnull)
time.sleep(GEM5_START_TIMEOUT)
def runtests(self, target):
nclients = siteconfig.get('HTTPERF_MAXCLIENTS')
firstrun = True
totalrate = HTTPERF_STARTRATE
while True:
if firstrun:
firstrun = False
else:
# sleep a moment to let things settle down between runs
debug.verbose('sleeping between httperf runs')
time.sleep(HTTPERF_SLEEPTIME)
# compute rate and total number of connections for each client
rate = totalrate / nclients
nconns = HTTPERF_DURATION * rate
debug.log('starting httperf: %d clients, %d conns, rate %d (%d per client)' %
(nclients, nconns, totalrate, rate))
self._runtest(target, nclients, nconns, rate)
# decide whether to keep going...
results = self._process_run(self.nruns)
if not results.passed():
debug.log('previous test failed, stopping')
break
elif results.request_rate < (0.9 * results.connect_rate):
debug.log('request rate below 90% of connect rate, stopping')
break
elif results.reply_rate < (0.9 * results.request_rate):
debug.log('reply rate below 90% of request rate, stopping')
break
else:
totalrate += HTTPERF_RATEINCREMENT
continue
def getpage_stress(self, server, page, count):
debug.verbose('requesting http://%s/%s' % (server, page))
failure_count = 0;
#c = httplib.HTTPConnection(server, timeout=WEBSERVER_TIMEOUT)
for i in range(count):
try:
c = httplib.HTTPConnection(server, timeout=WEBSERVER_TIMEOUT)
c.request('GET', '/' + page)
r = c.getresponse()
if (r.status / 100) != 2 :
print "HTTP request failed for %d" % (i)
assert((r.status / 100) == 2) # check for success response
# Reset failure count after sucessful retrival
failure_count = 0
c.close()
except:
print "HTTP request failed for %d, (failure count %d)" % (i,
failure_count)
failure_count = failure_count + 1
if failure_count >= 3:
print "HTTP request failed for 3 successive times."
print "Giving up for %d, (failure count %d)" % (i,
failure_count)
raise
#c.close()
debug.verbose('server replied %s %s for %d times' % (r.status, r.reason, count))
def collect_data(self, machine):
fh = machine.get_output()
while True:
try:
line = self._readline(fh)
except TimeoutError as e:
if self.boot_phase:
if self.boot_attempts < MAX_BOOT_ATTEMPTS:
yield BOOT_TIMEOUT_LINE_RETRY
self.reboot(machine)
continue
else:
yield BOOT_TIMEOUT_LINE_FAIL
else:
yield TEST_TIMEOUT_LINE
debug.verbose("timeout encountered in collect_data");
raise e
yield line
if not self.boot_phase:
self.process_line(line)
if self.is_finished(line):
debug.verbose("is_finished returned true for line %s" % line)
break
elif self.is_booted(line):
self.boot_phase = False
self.set_timeout(self.test_timeout_delta)
self.process_line(line)
def make_results_dir(options, build, machine, test):
# Create a unique directory for the output from this test
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
dirname = '-'.join([test.name, build.name, machine.name, timestamp])
path = os.path.join(options.resultsdir, str(datetime.datetime.now().year), dirname)
debug.verbose('create result directory %s' % path)
os.makedirs(path)
return path
def process_line(self, line):
m = re.match(r'Interface up! IP address (\d+\.\d+\.\d+\.\d+)', line)
if m:
self.ip = m.group(1)
elif self.ip and 'Starting webserver' in line:
debug.verbose("Running the tests")
self.runtests(self.ip)
self.finished = True
def _kill_child(self):
# terminate child if running
if self.child:
try:
os.kill(self.child.pid, signal.SIGTERM)
except OSError, e:
debug.verbose("Caught OSError trying to kill child: %r" % e)
except Exception, e:
debug.verbose("Caught exception trying to kill child: %r" % e)
def _cleanup_ipbenchd(self, user, host):
# run a remote killall to get rid of ipbenchd
ssh_dest = '%s@%s' % (user, host)
remotecmd = 'killall -q python'
cmd = ['ssh'] + siteconfig.get('SSH_ARGS').split() + [ssh_dest, remotecmd]
debug.verbose('killing ipbenchd on %s' % host)
retcode = subprocess.call(cmd)
if retcode != 0:
debug.warning('failed to killall python on %s!' % host)
def set_timeout(self, delta=DEFAULT_TEST_TIMEOUT):
self.test_timeout_delta = delta
if not self.boot_phase:
if delta:
debug.verbose('setting timeout for %s' % delta)
self.timeout = datetime.datetime.now() + delta
else:
debug.verbose('cancelling timeout')
self.timeout = None
def unlock(self):
if self.lockprocess is None:
return # noop
debug.verbose('quitting console process (%d)' % self.lockprocess.pid)
# os.kill(self.lockprocess.pid, signal.SIGTERM)
os.write(self.masterfd, "\x05c.")
self.lockprocess.wait()
self.lockprocess = None
self.masterfd = None
def runtests(self, server):
stress_counter = WEBSERVER_STRESS_COUNTER
self.testlog = open(os.path.join(self.testdir, TEST_LOG_NAME), 'w')
for f in WEBSERVER_TEST_FILES:
self.dotest(self.getpage, (server, f))
debug.verbose("Running stresstest: (%d GET %s)" %
(stress_counter, str(f)))
self.dotest(self.getpage_stress, (server, f, stress_counter))
self.testlog.close()
def interact(self):
self.wait_for_fish()
self.console.expect("On core %s" % self.core)
# Park
debug.verbose("Park OSNode from %s on %s." % (self.core, self.target_core))
self.console.sendline("corectrl park %s %s" % (self.core, self.target_core))
self.wait_for_prompt()
self.console.expect("On core %s" % self.target_core)
def interact(self):
self.wait_for_fish()
debug.verbose("Running corectrl lskcb")
self.console.sendline("corectrl lskcb")
self.console.expect("KCB 1:")
self.wait_for_prompt()
debug.verbose("Running corectrl lscpu")
self.console.sendline("corectrl lscpu")
self.console.expect("CPU 0:")
self.wait_for_prompt()
def _make_build_dir(self, build_dir=None):
if build_dir is None:
build_dir = os.path.join(self.options.buildbase, self.name.lower())
self.build_dir = build_dir
debug.verbose('creating build directory %s' % build_dir)
try:
os.makedirs(build_dir)
except OSError, e:
if e.errno == errno.EEXIST:
debug.log("reusing existing build in directory %s" % build_dir)
else:
raise
def reboot(self):
self._kill_child()
cmd = self._get_cmdline()
debug.verbose('starting "%s"' % ' '.join(cmd))
import pty
(self.masterfd, slavefd) = pty.openpty()
self.child = subprocess.Popen(cmd, close_fds=True,
stdout=slavefd,
stdin=slavefd)
os.close(slavefd)
# open in binary mode w/o buffering
self.qemu_out = os.fdopen(self.masterfd, 'rb', 0)
def reboot(self):
self._kill_child()
cmd = self._get_cmdline()
debug.verbose('starting "%s" in gem5.py:reboot' % ' '.join(cmd))
devnull = open('/dev/null', 'w')
# remove ubuntu chroot from environment to make sure gem5 finds the
# right shared libraries
import os
env = dict(os.environ)
if 'LD_LIBRARY_PATH' in env:
del env['LD_LIBRARY_PATH']
self.child = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull, env=env)
time.sleep(GEM5_START_TIMEOUT)
def getpage(self, server, page):
debug.verbose('requesting http://%s/%s' % (server, page))
c = httplib.HTTPConnection(server, timeout=WEBSERVER_TIMEOUT)
c.request('GET', '/' + page)
r = c.getresponse()
debug.verbose('server replied %s %s' % (r.status, r.reason))
assert((r.status / 100) == 2) # check for success response
try:
local_path = siteconfig.get('WEBSERVER_LOCAL_PATH')
except AttributeError:
local_path = None
local = os.path.join(local_path, page) if local_path else None
if local and os.path.isfile(local) and os.access(local, os.R_OK):
debug.verbose('comparing content to %s' % local)
l = open(local, 'r')
# read from both files and compare
CHUNKSIZE = 4096
while True:
remote_data = r.read(CHUNKSIZE)
local_data = l.read(CHUNKSIZE)
if remote_data != local_data:
print "Remote and local data did not match:"
print "Remote data\n"
print remote_data
print "Local data\n"
print local_data
assert(remote_data == local_data)
if len(local_data) < CHUNKSIZE:
break
debug.verbose('contents matched for %s' % local)
c.close()
def dotest(self, func, args):
exception = None
r = None
try:
r = func(*args)
except Exception as e:
exception = e
s = "Test: %s%s\t%s\n" % (func.__name__, str(args), "FAIL" if exception else "PASS")
if exception:
debug.verbose("Exception while running test: %s\n" % traceback.format_exc())
s += "Error was: %s\n" % traceback.format_exc()
self.testlog.write(s)
return r
def __rackpower(self, arg):
retries = 3
failed = False
while retries > 0:
try:
debug.checkcmd([RACKPOWER, arg, self.get_machine_name()])
except subprocess.CalledProcessError:
debug.warning("rackpower %s %s failed" %
(arg, self.get_machine_name()))
failed = True
if retries > 0:
debug.verbose("retrying...")
retries -= 1
if not failed:
break
def dotest(self, func, args):
exception = None
r = None
try:
r = func(*args)
except Exception as e:
exception = e
s = 'Test: %s%s\t%s\n' % (func.__name__, str(args),
'FAIL' if exception else 'PASS')
if exception:
debug.verbose('Exception while running test: %s\n'
% traceback.format_exc())
s += 'Error was: %s\n' % traceback.format_exc()
self.testlog.write(s)
return r
def write_description(options, checkout, build, machine, test, path):
debug.verbose('write description file')
f = open(os.path.join(path, 'description.txt'), 'w')
f.write('test: %s\n' % test.name)
f.write('revision: %s\n' % checkout.describe())
f.write('build: %s\n' % build.name)
f.write('machine: %s\n' % machine.name)
f.write('start time: %s\n' % datetime.datetime.now())
f.write('user: %s\n' % getpass.getuser())
if options.comment:
f.write('\n' + options.comment + '\n')
f.close()
diff = checkout.changes()
if diff:
with open(os.path.join(path, 'changes.patch'), 'w') as f:
f.write(diff)
def cleanup(self):
"""perform cleanup if necessary"""
self.logfile.close()
if self.proc is None or self.proc.poll() == 0:
return # clean exit
if self.proc.returncode:
debug.warning("httperf: SSH to %s exited with error %d" % (self.host, self.proc.returncode))
else: # kill SSH if still up
debug.warning("httperf: killing SSH child for %s" % self.host)
self.proc.terminate()
self.proc.wait()
# run a remote killall to get rid of any errant httperfs
debug.verbose("killing any errant httperfs on %s" % self.host)
p = self._launchssh("killall -q %s" % self.httperf_path)
retcode = p.wait()
if retcode != 0:
debug.warning("failed to killall httperf on %s!" % self.host)
def interact(self):
self.wait_for_fish()
# wait for app
self.console.expect("On core %s" % self.core)
debug.verbose("Stopping core %s." % self.core)
self.console.sendline("corectrl stop %s" % self.core)
# Stop core
debug.verbose("Wait until core is down.")
self.console.expect("Core %s stopped." % self.core)
# cannot wait for prompt here, as new cleanup routine will wait for
# answer from monitor on stopped core.
#self.wait_for_prompt()
# Make sure app is no longer running
i = self.console.expect(["On core %s" % self.core, pexpect.TIMEOUT], timeout=10)
if i == 0:
raise Exception("periodicprint still running, did we not shut-down the core?")
def main(dirs):
for dirname in dirs:
debug.log('reprocessing %s' % dirname)
debug.verbose('parse %s/description.txt for test' % dirname)
testname = test = None
f = open(os.path.join(dirname, 'description.txt'), 'r')
for line in f:
m = re.match(r'test:\s+(.*)', line)
if m:
testname = m.group(1)
break
f.close()
if not testname:
debug.error('unable to parse description for %s, skipped' % dirname)
continue
debug.verbose('locate test "%s"' % testname)
for t in tests.all_tests:
if t.name.lower() == testname.lower():
test = t(None) # XXX: dummy options
if not test:
debug.error('unknown test "%s" in %s, skipped' % (testname, dirname))
continue
debug.verbose('reprocess results')
harness.process_results(test, dirname)
def process_output(self, test, path):
"""Process raw.txt and return array of output lines that begins with grubs
output, avoids having encoding issues when generating other report files"""
raw_file_name = os.path.join(path, self.RAW_FILE_NAME)
if os.path.exists(raw_file_name):
idx = 0
with open(raw_file_name, 'r') as rf:
lines = rf.readlines()
for idx, line in enumerate(lines):
if line.strip() == "root (nd)" or \
line.strip().startswith("Kernel starting at address"):
break
if idx == len(lines)-1:
debug.verbose('magic string "root (nd)" or "Kernel starting at address" not found, assuming no garbage in output')
idx=0
return [ unicode(self._clean_line(l), errors='replace') for l in lines[idx:] ]
# file did not exist
return ["could not open %s to process test output" % raw_file_name]
def interact(self):
self.wait_for_fish()
# wait for app
self.console.expect("On core %s" % self.core)
debug.verbose("Stopping core %s." % self.core)
self.console.sendline("corectrl stop %s" % self.core)
# Stop core
debug.verbose("Wait until core is down.")
self.console.expect("Core %s stopped." % self.core)
# cannot wait for prompt here, as new cleanup routine will wait for
# answer from monitor on stopped core.
#self.wait_for_prompt()
# Make sure app is no longer running
i = self.console.expect(["On core %s" % self.core, pexpect.TIMEOUT],
timeout=10)
if i == 0:
raise Exception(
"periodicprint still running, did we not shut-down the core?")
def interact(self):
self.wait_for_fish()
self.console.expect("On core %s" % self.core)
self.console.expect("On core %s" % self.core)
# Park
debug.verbose("Park KCB %s on core %s." % (self.core, self.parking_core))
self.console.sendline("corectrl park %s %s\n" % (self.core, self.parking_core))
self.wait_for_prompt()
self.console.expect("On core %s" % self.parking_core)
self.console.expect("On core %s" % self.parking_core)
# Unpark
debug.verbose("Unpark KCB %s from core %s." % (self.core, self.parking_core))
self.console.sendline("corectrl unpark %s\n" % (self.core))
self.wait_for_prompt()
# Reboot home core with kcb
self.console.expect("On core %s" % self.core)
self.console.expect("On core %s" % self.core)
def shutdown(self):
debug.verbose('FVP:shutdown requested')
debug.verbose('terminating FVP')
if not self.child is None:
try:
self.child.terminate()
except OSError, e:
debug.verbose("Error when trying to terminate FVP: %r" % e)
def shutdown(self):
debug.verbose('Simulator:shutdown requested');
debug.verbose('terminating simulator')
if not self.child is None:
try:
self.child.terminate()
except OSError, e:
debug.verbose("Error when trying to terminate simulator: %r" % e)
def set_bootmodules(self, modules):
# write menu.lst in build directory
debug.verbose("writing menu.lst in build directory")
menulst_fullpath = os.path.join(self.options.buildbase, "platforms",
"arm", "menu.lst.armv7_a15ve")
debug.verbose("writing menu.lst in build directory: %s" %
menulst_fullpath)
self._write_menu_lst(modules.get_menu_data("/"), menulst_fullpath)
debug.verbose("building proper gem5 image")
debug.checkcmd(["make", self.imagename], cwd=self.options.buildbase)
def shutdown(self):
debug.verbose('gem5:shutdown requested')
debug.verbose('terminating gem5')
self.child.terminate()
debug.verbose('terminating telnet')
self.telnet.terminate()
# try to cleanup tftp tree if needed
if self.tftp_dir and os.path.isdir(self.tftp_dir):
shutil.rmtree(self.tftp_dir, ignore_errors=True)
self.tftp_dir = None
def set_bootmodules(self, modules):
super(FVPMachineARMv7SingleCore, self).set_bootmodules(modules)
debug.verbose("writing menu.lst in build directory")
menulst_fullpath = os.path.join(self.builddir, "platforms", "arm",
"menu.lst.armv7_a9ve")
debug.verbose("writing menu.lst in build directory: %s" %
menulst_fullpath)
self._write_menu_lst(modules.get_menu_data("/"), menulst_fullpath)
debug.verbose("building proper FVP image")
debug.checkcmd(["make", IMAGE_NAME], cwd=self.builddir)
class ARMSimulatorOperations(MachineOperations):
def __init__(self, machine):
super(ARMSimulatorOperations, self).__init__(machine)
self.child = None
self.telnet = None
self.tftp_dir = None
self.simulator_start_timeout = 5 # seconds
def setup(self):
pass
def force_write(self, consolectrl):
pass
def lock(self):
pass
def unlock(self):
pass
def get_free_port(self):
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', 0))
# extract port from addrinfo
self.telnet_port = s.getsockname()[1]
s.close()
def _get_cmdline(self):
raise NotImplementedError
def _kill_child(self):
# terminate child if running
if self.child:
try:
os.kill(self.child.pid, signal.SIGTERM)
except OSError, e:
debug.verbose("Caught OSError trying to kill child: %r" % e)
except Exception, e:
debug.verbose("Caught exception trying to kill child: %r" % e)
try:
self.child.wait()
except Exception, e:
debug.verbose(
"Caught exception while waiting for child: %r" % e)
def _setup_harness_dir(self, build, machine):
dest_dir = machine.get_tftp_dir()
debug.verbose('installing to %s' % dest_dir)
if os.access(dest_dir, os.F_OK):
debug.verbose('clearing out %s' % dest_dir)
for e in os.listdir(dest_dir):
p = os.path.join(dest_dir, e)
if os.path.isdir(p):
shutil.rmtree(p, ignore_errors=True)
elif not e.startswith('.nfs'):
os.unlink(p)
else:
debug.verbose('creating %s' % dest_dir)
os.makedirs(dest_dir)
return dest_dir
def set_bootmodules(self, modules):
# write menu.lst in build directory
debug.verbose("writing menu.lst in build directory")
menulst_fullpath = os.path.join(self.options.builds[0].build_dir,
"platforms", "arm", "menu.lst.armv8_base")
debug.verbose("writing menu.lst in build directory: %s" %
menulst_fullpath)
self._write_menu_lst(modules.get_menu_data("/"), menulst_fullpath)
debug.checkcmd(["make"] + modules.get_build_targets(), cwd=self.options.builds[0].build_dir)
debug.verbose("building proper FVP image")
efi = efiimage.EFIImage(self.kernel_img, 200)
efi.create()
for module in modules.get_build_targets():
efi.addFile(os.path.join(self.options.builds[0].build_dir, module), module)
efi.writeFile("startup.nsh", "Hagfish.efi hagfish.cfg")
efi.addFile("/home/netos/tftpboot/Hagfish.efi", "Hagfish.efi")
efi.addFile(menulst_fullpath, "hagfish.cfg")
def collect_data(self, machine):
fh = machine.get_output()
if fh is None:
yield TEST_NO_OUTPUT_LINE
return
while True:
try:
line = self._readline(fh)
except TimeoutError as e:
if self.boot_phase:
if self.boot_attempts < MAX_BOOT_ATTEMPTS:
yield BOOT_TIMEOUT_LINE_RETRY
self.reboot(machine)
continue
else:
yield BOOT_TIMEOUT_LINE_FAIL
else:
yield TEST_TIMEOUT_LINE
debug.verbose("timeout encountered in collect_data");
raise e
except EOFError as e:
debug.verbose("got EOF from sub-process")
break
yield line
if not self.boot_phase:
self.process_line(line)
if self.is_finished(line):
debug.verbose("is_finished returned true for line %s" % line)
# Read remaining lines from console until it blocks
if self.read_after_finished:
for x in self._read_until_block(fh):
self.process_line(x)
yield x
break
elif self.is_booted(line):
self.boot_phase = False
self.set_timeout(self.test_timeout_delta)
self.process_line(line)
def parse_args():
p = optparse.OptionParser(
usage='Usage: %prog [options] SOURCEDIR RESULTDIR',
description='Barrelfish regression/benchmark harness')
g = optparse.OptionGroup(p, 'Basic options')
g.add_option('-b',
'--build',
action='append',
dest='buildspecs',
metavar='BUILD',
help='build types to perform [default: test]')
g.add_option('-B',
'--buildbase',
dest='buildbase',
metavar='DIR',
help='places builds under DIR [default: SOURCEDIR/builds]')
g.add_option('-e',
'--existingbuild',
dest='existingbuild',
metavar='DIR',
help='existing build directory (may not be used with -b)')
g.add_option('-m',
'--machine',
action='append',
dest='machinespecs',
metavar='MACHINE',
help='victim machines to use')
g.add_option('-t',
'--test',
action='append',
dest='testspecs',
metavar='TEST',
help='tests/benchmarks to run')
g.add_option('-c',
'--comment',
dest='comment',
help='comment to store with all collected data')
g.add_option('-x',
'--xml',
dest='xml',
action='store_true',
default=False,
help='output summary of tests in Junit XML format')
p.add_option_group(g)
g = optparse.OptionGroup(p, 'Debugging options')
g.add_option('-L',
'--listall',
action='store_true',
dest='listall',
help='list available builds, machines and tests')
debug.addopts(g, 'debuglevel')
g.add_option('-k',
'--keepgoing',
action='store_true',
dest='keepgoing',
help='attempt to continue on errors')
p.add_option_group(g)
p.set_defaults(debuglevel=debug.NORMAL)
options, args = p.parse_args()
debug.current_level = options.debuglevel
if options.listall:
list_all()
sys.exit(0)
if len(args) != 2:
p.error('source and results directories must be specified')
options.sourcedir, options.resultsdir = args
# determine default buildbase if needed
if options.buildbase is None:
options.buildbase = os.path.join(options.sourcedir, 'builds')
# check validity of source and results dirs
if not os.path.isdir(os.path.join(options.sourcedir, 'hake')):
p.error('invalid source directory %s' % options.sourcedir)
if not (os.path.isdir(options.resultsdir)
and os.access(options.resultsdir, os.W_OK)):
p.error('invalid results directory %s' % options.resultsdir)
if options.xml and not have_junit_xml:
p.error('--xml requires junit-xml.\n'
'Please install junit-xml through pip or easy_install')
# resolve and instantiate all builds
def _lookup(spec, classes, nameFn=lambda c: c.name.lower()):
spec = spec.lower()
return [c for c in classes if fnmatch.fnmatch(nameFn(c), spec)]
if options.existingbuild:
if options.buildspecs:
p.error('existing build directory cannot be used together'
' with build types (-b)')
options.builds = [builds.existingbuild(options, options.existingbuild)]
options.buildbase = options.existingbuild
else:
options.builds = []
if not options.buildspecs:
options.buildspecs = ['test']
for spec in options.buildspecs:
matches = _lookup(spec, builds.all_builds)
if matches == []:
p.error('no builds match "%s" (try -L for a list)' % spec)
options.builds.extend(
[b for b in matches if b not in options.builds])
options.builds = [b(options) for b in options.builds]
# resolve and instantiate all machines
if options.machinespecs is None:
p.error('no machines specified')
options.machines = []
for spec in options.machinespecs:
matches = _lookup(spec,
MachineFactory.machineFactories,
nameFn=lambda fac: fac.lower())
if matches == []:
p.error('no machines match "%s" (try -L for a list)' % spec)
options.machines.extend(
[m for m in matches if m not in options.machines])
options.machines = [
MachineFactory.createMachineByName(m, options)
for m in options.machines
]
# resolve and instantiate all tests
if options.testspecs:
options.tests = []
for spec in options.testspecs:
matches = _lookup(spec, tests.all_tests)
if matches == []:
p.error('no tests match "%s" (try -L for a list)' % spec)
options.tests.extend(
[t for t in matches if t not in options.tests])
else:
p.error('no tests specified (try -t memtest if unsure)')
options.tests = [t(options) for t in options.tests]
debug.verbose('Host: ' + gethostname())
debug.verbose('Builds: ' + ', '.join([b.name for b in options.builds]))
debug.verbose('Machines: ' +
', '.join([m.getName() for m in options.machines]))
debug.verbose('Tests: ' + ', '.join([t.name for t in options.tests]))
return options
def wait_for_fish(self):
debug.verbose("Waiting for fish.")
self.console.expect("fish v0.2 -- pleased to meet you!",
timeout=self.test_timeout)
self.wait_for_prompt()
def cleanup(self, machine):
tftp_dir = machine.get_tftp_dir()
machine.shutdown()
machine.unlock()
debug.verbose('removing %s' % tftp_dir)
shutil.rmtree(tftp_dir, ignore_errors=True)
def get_tftp_dir(self):
if self.tftp_dir is None:
debug.verbose('creating temporary directory for Gem5 files')
self.tftp_dir = tempfile.mkdtemp(prefix='harness_gem5_')
debug.verbose('Gem5 install directory is %s' % self.tftp_dir)
return self.tftp_dir
def _set_menu_lst(self, relpath):
ip_menu_name = os.path.join(TFTP_PATH, "menu.lst." + self.get_ip())
debug.verbose('relinking %s to %s' % (ip_menu_name, relpath))
os.remove(ip_menu_name)
os.symlink(relpath, ip_menu_name)
def process_results(self, test, path):
# open raw file for input processing
raw_file_name = os.path.join(path, self.RAW_FILE_NAME)
debug.verbose('open %s for raw input' % raw_file_name)
raw_file = open(raw_file_name, 'r')
try:
results = test.process_data(path, raw_file)
finally:
raw_file.close()
if not results:
debug.verbose('no results')
return True # no results, assume success
retval = True # everything OK
# Process raw.txt and make a bootlog.txt that begins with grubs or
# Barrelfish's output, avoids having encoding issues when viewing logfiles
boot_file_name = os.path.join(path, self.BOOT_FILE_NAME)
if os.path.exists(raw_file_name):
idx = 0
with open(raw_file_name, 'r') as rf:
lines = rf.readlines()
for idx, line in enumerate(lines):
if line.strip() == "root (nd)" or \
line.strip().startswith("Kernel starting at address"):
break
if idx > 0:
with open(boot_file_name, 'w') as wf:
wf.writelines(lines[idx:])
else:
debug.verbose(
'Magic string root (nd) not found, do not write bootlog.txt'
)
else:
debug.verbose(
'No file named %s exists. Do not create bootlog.txt.' %
raw_file_name)
# if a single result, turn it into a list
if not isinstance(results, types.ListType):
results = [results]
for result in results:
# see if it passed
try:
passed = result.passed()
except NotImplementedError:
passed = None
if passed is False:
debug.log('Test %s FAILED %s' %
(test.name, '(' + result.reason() + ')'))
retval = False
elif passed:
debug.verbose('Test %s PASSED' % test.name)
# write it to a file
name = result.name if result.name else 'results'
data_file_name = os.path.join(path, name + '.dat')
debug.verbose('create %s for processed output' % data_file_name)
data_file = open(data_file_name, 'w')
try:
result.to_file(data_file)
data_file.close()
except NotImplementedError:
debug.verbose('no processed output, remove %s' %
data_file_name)
data_file.close()
os.remove(data_file_name)
return retval
def __usbboot(self):
imagename = os.path.join(self.builddir, IMAGE_NAME)
debug.verbose("Usbbooting pandaboard; press reset")
debug.checkcmd(["usbboot", imagename])
def get_tftp_dir(self):
if self.tftp_dir is None:
debug.verbose('creating temporary directory for QEMU TFTP files')
self.tftp_dir = tempfile.mkdtemp(prefix='harness_qemu_')
debug.verbose('QEMU TFTP directory is %s' % self.tftp_dir)
return self.tftp_dir
class FVPMachineBase(Machine):
def __init__(self, options):
super(FVPMachineBase, self).__init__(options)
self.child = None
self.telnet = None
self.tftp_dir = None
self.options = options
def get_buildall_target(self):
return "VExpressEMM-A9"
def get_coreids(self):
return range(0, self.get_ncores())
def get_tickrate(self):
return None
def get_boot_timeout(self):
return 120
def get_test_timeout(self):
# 15 mins
return 15 * 60
def get_machine_name(self):
return self.name
def force_write(self, consolectrl):
pass
def get_tftp_dir(self):
if self.tftp_dir is None:
debug.verbose('Creating temporary directory for FVP files')
self.tftp_dir = tempfile.mkdtemp(prefix='harness_fvp_')
debug.verbose('FVP install directory is %s' % self.tftp_dir)
return self.tftp_dir
# Use menu.lst in hake/menu.lst.arm_fvp
def _write_menu_lst(self, data, path):
pass
def set_bootmodules(self, modules):
pass
def lock(self):
pass
def unlock(self):
pass
def setup(self, builddir=None):
self.builddir = builddir
def _get_cmdline(self):
raise NotImplementedError
def get_kernel_args(self):
# Fixed virtual platform has 100MHz clock that is not discoverable
return ["periphclk=100000000", "consolePort=0"]
def _kill_child(self):
# terminate child if running
if self.child:
try:
os.kill(self.child.pid, signal.SIGTERM)
except OSError, e:
debug.verbose("Caught OSError trying to kill child: %r" % e)
except Exception, e:
debug.verbose("Caught exception trying to kill child: %r" % e)
try:
self.child.wait()
except Exception, e:
debug.verbose("Caught exception while waiting for child: %r" %
e)
def _write_menu_lst(self, data, path):
debug.verbose('writing %s' % path)
debug.debug(data)
with open(path, 'w') as f:
f.write(data)
env = dict(os.environ)
env['ARMLMD_LICENSE_FILE'] = FVP_LICENSE
self.child = \
subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=devnull, env=env)
time.sleep(FVP_START_TIMEOUT)
def shutdown(self):
debug.verbose('FVP:shutdown requested')
debug.verbose('terminating FVP')
if not self.child is None:
try:
self.child.terminate()
except OSError, e:
debug.verbose("Error when trying to terminate FVP: %r" % e)
debug.verbose('closing telnet connection')
if not self.telnet is None:
self.output.close()
self.telnet.close()
# try to cleanup tftp tree if needed
if self.tftp_dir and os.path.isdir(self.tftp_dir):
shutil.rmtree(self.tftp_dir, ignore_errors=True)
self.tftp_dir = None
def get_output(self):
# wait a bit to give FVP time to listen for a telnet connection
if self.child.poll() != None: # Check if child is down
print 'FVP is down, return code is %d' % self.child.returncode
return None
# use telnetlib
import telnetlib
def __usbboot(self):
debug.verbose("Usbbooting pandaboard; press reset")
debug.verbose("build dir: %s" % self.builddir)
debug.checkcmd(["make", "usbboot_panda"], cwd=self.builddir)
def get_tftp_dir(self):
if self.tftp_dir is None:
debug.verbose('Creating temporary directory for FVP files')
self.tftp_dir = tempfile.mkdtemp(prefix='harness_fvp_')
debug.verbose('FVP install directory is %s' % self.tftp_dir)
return self.tftp_dir
def setup(self, build, machine, testdir):
super(MultithreadedWaitsetTest, self).setup(build, machine, testdir)
self.test_timeout_delta *= 3
debug.verbose("%s: increasing test timeout delta by factor 3: new = %s" %
(self.name, self.test_timeout_delta))
def __init__(self, options, build_dir):
super(ExistingBuild, self).__init__(options)
debug.verbose('using existing build directory %s' % build_dir)
self.build_dir = build_dir
def set_bootmodules(self, modules):
menulst_fullpath = os.path.join(self.builddir, "platforms", "arm",
"menu.lst.armv7_pandaboard")
self._write_menu_lst(modules.get_menu_data("/"), menulst_fullpath)
debug.verbose("building proper pandaboard image")
debug.checkcmd(["make", IMAGE_NAME], cwd=self.builddir)
def _write_menu_lst(self, data, path):
debug.verbose('writing %s' % path)
debug.debug(data)
f = open(path, 'w')
f.write(data)
f.close()
class ARMSimulatorBase(ARMMachineBase):
def __init__(self, options):
super(ARMSimulatorBase, self).__init__(options)
self.child = None
self.telnet = None
self.tftp_dir = None
self.simulator_start_timeout = 5 # seconds
def setup(self):
pass
def get_coreids(self):
return range(0, self.get_ncores())
def get_tickrate(self):
return None
def get_boot_timeout(self):
"""Default boot timeout for ARM simulators: 2min"""
return 120
def get_test_timeout(self):
"""Default test timeout for ARM simulators: 10min"""
return 10 * 60
def get_machine_name(self):
return self.name
def get_bootarch(self):
raise NotImplementedError
def get_platform(self):
raise NotImplementedError
def force_write(self, consolectrl):
pass
def lock(self):
pass
def unlock(self):
pass
def get_free_port(self):
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', 0))
# extract port from addrinfo
self.telnet_port = s.getsockname()[1]
s.close()
def _get_cmdline(self):
raise NotImplementedError
def _kill_child(self):
# terminate child if running
if self.child:
try:
os.kill(self.child.pid, signal.SIGTERM)
except OSError, e:
debug.verbose("Caught OSError trying to kill child: %r" % e)
except Exception, e:
debug.verbose("Caught exception trying to kill child: %r" % e)
try:
self.child.wait()
except Exception, e:
debug.verbose("Caught exception while waiting for child: %r" %
e)
