def start(self, out):
# Setup environment variables
logDir = os.path.join(self.fwroot, self.benchmarker.full_results_directory(), 'logs', self.name.lower())
bash_functions_path= os.path.join(self.fwroot, 'toolset/setup/linux/bash_functions.sh')
os.environ['TROOT'] = self.directory
os.environ['IROOT'] = self.install_root
os.environ['DBHOST'] = socket.gethostbyname(self.database_host)
os.environ['LOGDIR'] = logDir
os.environ['MAX_CONCURRENCY'] = str(max(self.benchmarker.concurrency_levels))
# Always ensure that IROOT exists
if not os.path.exists(self.install_root):
os.mkdir(self.install_root)
if not os.path.exists(os.path.join(self.install_root,"TFBReaper")):
subprocess.check_call(['gcc',
'-std=c99',
'-o%s/TFBReaper' % self.install_root,
os.path.join(self.fwroot,'toolset/setup/linux/TFBReaper.c')],
stderr=out, stdout=out)
# Check that the client is setup
if not os.path.exists(os.path.join(self.install_root, 'client.installed')):
print("\nINSTALL: Installing client software\n")
# TODO: hax; should dynamically know where this file is
with open (self.fwroot + "/toolset/setup/linux/client.sh", "r") as myfile:
remote_script=myfile.read()
print("\nINSTALL: %s" % self.benchmarker.client_ssh_string)
p = subprocess.Popen(self.benchmarker.client_ssh_string.split(" ") + ["bash"], stdin=subprocess.PIPE)
p.communicate(remote_script)
returncode = p.returncode
if returncode != 0:
self.__install_error("status code %s running subprocess '%s'." % (returncode, self.benchmarker.client_ssh_string))
print("\nINSTALL: Finished installing client software\n")
subprocess.check_call('touch client.installed', shell=True, cwd=self.install_root, executable='/bin/bash')
# Run the module start inside parent of TROOT
# - we use the parent as a historical accident, a number of tests
# refer to their TROOT maually still
previousDir = os.getcwd()
os.chdir(os.path.dirname(self.troot))
logging.info("Running setup module start (cwd=%s)", self.directory)
command = 'bash -exc "source %s && source %s.sh"' % (
bash_functions_path,
os.path.join(self.troot, self.setup_file))
debug_command = '''\
export FWROOT=%s && \\
export TROOT=%s && \\
export IROOT=%s && \\
export DBHOST=%s && \\
export LOGDIR=%s && \\
export MAX_CONCURRENCY=%s && \\
cd %s && \\
%s/TFBReaper "bash -exc \\\"source %s && source %s.sh\\\"''' % (self.fwroot,
self.directory,
self.install_root,
socket.gethostbyname(self.database_host),
logDir,
max(self.benchmarker.concurrency_levels),
self.directory,
self.install_root,
bash_functions_path,
os.path.join(self.troot, self.setup_file))
logging.info("To run %s manually, copy/paste this:\n%s", self.name, debug_command)
def tee_output(prefix, line):
# Needs to be one atomic write
# Explicitly use UTF-8 as it's the most common framework output
# TODO improve encoding handling
line = prefix.encode('utf-8') + line
# Log to current terminal
sys.stdout.write(line)
sys.stdout.flush()
# logging.error("".join([prefix, line]))
out.write(line)
out.flush()
# Start the setup.sh command
p = subprocess.Popen(["%s/TFBReaper" % self.install_root,command],
cwd=self.directory,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
nbsr = setup_util.NonBlockingStreamReader(p.stdout,
"%s: %s.sh and framework processes have terminated" % (self.name, self.setup_file))
# Set a limit on total execution time of setup.sh
timeout = datetime.now() + timedelta(minutes = 105)
time_remaining = timeout - datetime.now()
# Need to print to stdout once every 10 minutes or Travis-CI will abort
travis_timeout = datetime.now() + timedelta(minutes = 5)
# Flush output until setup.sh work is finished. This is
# either a) when setup.sh exits b) when the port is bound
# c) when we run out of time. Note that 'finished' doesn't
# guarantee setup.sh process is dead - the OS may choose to make
# setup.sh a zombie process if it still has living children
#
# Note: child processes forked (using &) will remain alive
# after setup.sh has exited. The will have inherited the
# stdout/stderr descriptors and will be directing their
# output to the pipes.
#
prefix = "Setup %s: " % self.name
while (p.poll() is None
and not self.benchmarker.is_port_bound(self.port)
and not time_remaining.total_seconds() < 0):
# The conditions above are slow to check, so
# we will delay output substantially if we only
# print one line per condition check.
# Adding a tight loop here mitigates the effect,
# ensuring that most of the output directly from
# setup.sh is sent to tee_output before the outer
# loop exits and prints things like "setup.sh exited"
#
for i in xrange(10):
try:
line = nbsr.readline(0.05)
if line:
tee_output(prefix, line)
# Reset Travis-CI timer
travis_timeout = datetime.now() + timedelta(minutes = 5)
except setup_util.EndOfStream:
tee_output(prefix, "Setup has terminated\n")
break
time_remaining = timeout - datetime.now()
if (travis_timeout - datetime.now()).total_seconds() < 0:
sys.stdout.write(prefix + 'Printing so Travis-CI does not time out\n')
sys.stdout.write(prefix + "Status: Poll: %s, Port %s bound: %s, Time Left: %s\n" % (
p.poll(), self.port, self.benchmarker.is_port_bound(self.port), time_remaining))
sys.stdout.flush()
travis_timeout = datetime.now() + timedelta(minutes = 5)
# Did we time out?
if time_remaining.total_seconds() < 0:
tee_output(prefix, "%s.sh timed out!! Aborting...\n" % self.setup_file)
p.kill()
return 1
# What's our return code?
# If setup.sh has terminated, use that code
# Otherwise, detect if the port was bound
tee_output(prefix, "Status: Poll: %s, Port %s bound: %s, Time Left: %s\n" % (
p.poll(), self.port, self.benchmarker.is_port_bound(self.port), time_remaining))
retcode = (p.poll() if p.poll() is not None else 0 if self.benchmarker.is_port_bound(self.port) else 1)
if p.poll() is not None:
tee_output(prefix, "%s.sh process exited naturally with %s\n" % (self.setup_file, p.poll()))
elif self.benchmarker.is_port_bound(self.port):
tee_output(prefix, "Bound port detected on %s\n" % self.port)
# Before we return control to the benchmarker, spin up a
# thread to keep an eye on the pipes in case the running
# framework uses stdout/stderr. Once all processes accessing
# the subprocess.PIPEs are dead, this thread will terminate.
# Use a different prefix to indicate this is the framework
# speaking
prefix = "Server %s: " % self.name
def watch_child_pipes(nbsr, prefix):
while True:
try:
line = nbsr.readline(60)
if line:
tee_output(prefix, line)
except setup_util.EndOfStream:
tee_output(prefix, "Framework processes have terminated\n")
return
watch_thread = Thread(target = watch_child_pipes,
args = (nbsr, prefix))
watch_thread.daemon = True
watch_thread.start()
logging.info("Executed %s.sh, returning %s", self.setup_file, retcode)
os.chdir(previousDir)
return retcode, p
def start(self, out):
# Setup environment variables
logDir = os.path.join(self.fwroot, self.benchmarker.latest_results_directory, 'logs', self.name.lower())
bash_functions_path= os.path.join(self.fwroot, 'toolset/setup/linux/bash_functions.sh')
setup_util.replace_environ(config='$FWROOT/config/benchmark_profile',
command='''\
export TROOT=%s && \
export IROOT=%s && \
export DBHOST=%s && \
export LOGDIR=%s && \
export MAX_THREADS=%s && \
export MAX_CONCURRENCY=%s \
''' % (
self.directory,
self.install_root,
self.database_host,
logDir,
self.benchmarker.threads,
max(self.benchmarker.concurrency_levels)))
# Always ensure that IROOT belongs to the runner_user
if not os.path.exists(self.install_root):
os.mkdir(self.install_root)
chown = "sudo chown -R %s:%s %s" % (self.benchmarker.runner_user,
self.benchmarker.runner_user, os.path.join(self.fwroot, self.install_root))
subprocess.check_call(chown, shell=True, cwd=self.fwroot, executable='/bin/bash')
# Run the module start inside parent of TROOT
# - we use the parent as a historical accident, a number of tests
# refer to their TROOT maually still
previousDir = os.getcwd()
os.chdir(os.path.dirname(self.troot))
logging.info("Running setup module start (cwd=%s)", self.directory)
# Run the start script for the test as the "testrunner" user
#
# `sudo` - Switching user requires superuser privs
# -u [username] The username
# -E Preserves the current environment variables
# -H Forces the home var (~) to be reset to the user specified
# `stdbuf` - Disable buffering, send output to python ASAP
# -o0 zero-sized buffer for stdout
# -e0 zero-sized buffer for stderr
# `bash` - Run the setup.sh script using bash
# -e Force bash to exit on first error
# -x Turn on bash tracing e.g. print commands before running
#
# Most servers do not output to stdout/stderr while serving
# requests so there is no performance hit from disabling
# output buffering. This disabling is necessary to
# a) allow TFB to show output in real time and b) avoid loosing
# output in the buffer when the testrunner processes are forcibly
# killed
#
# See http://www.pixelbeat.org/programming/stdio_buffering/
# See https://blogs.gnome.org/markmc/2013/06/04/async-io-and-python/
# See http://eyalarubas.com/python-subproc-nonblock.html
command = 'sudo -u %s -E -H stdbuf -o0 -e0 bash -exc "source %s && source %s.sh"' % (
self.benchmarker.runner_user,
bash_functions_path,
os.path.join(self.troot, self.setup_file))
debug_command = '''\
export FWROOT=%s && \\
export TROOT=%s && \\
export IROOT=%s && \\
export DBHOST=%s && \\
export LOGDIR=%s && \\
export MAX_THREADS=%s && \\
export MAX_CONCURRENCY=%s && \\
cd %s && \\
%s''' % (self.fwroot,
self.directory,
self.install_root,
self.database_host,
logDir,
self.benchmarker.threads,
max(self.benchmarker.concurrency_levels),
self.directory,
command)
logging.info("To run %s manually, copy/paste this:\n%s", self.name, debug_command)
def tee_output(prefix, line):
# Needs to be one atomic write
# Explicitly use UTF-8 as it's the most common framework output
# TODO improve encoding handling
line = prefix.encode('utf-8') + line
# Log to current terminal
sys.stdout.write(line)
sys.stdout.flush()
# logging.error("".join([prefix, line]))
out.write(line)
out.flush()
# Start the setup.sh command
p = subprocess.Popen(command, cwd=self.directory,
shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
nbsr = setup_util.NonBlockingStreamReader(p.stdout,
"%s: %s.sh and framework processes have terminated" % (self.name, self.setup_file))
# Set a limit on total execution time of setup.sh
timeout = datetime.now() + timedelta(minutes = 105)
time_remaining = timeout - datetime.now()
# Need to print to stdout once every 10 minutes or Travis-CI will abort
travis_timeout = datetime.now() + timedelta(minutes = 5)
# Flush output until setup.sh work is finished. This is
# either a) when setup.sh exits b) when the port is bound
# c) when we run out of time. Note that 'finished' doesn't
# guarantee setup.sh process is dead - the OS may choose to make
# setup.sh a zombie process if it still has living children
#
# Note: child processes forked (using &) will remain alive
# after setup.sh has exited. The will have inherited the
# stdout/stderr descriptors and will be directing their
# output to the pipes.
#
prefix = "Setup %s: " % self.name
while (p.poll() is None
and not self.benchmarker.is_port_bound(self.port)
and not time_remaining.total_seconds() < 0):
# The conditions above are slow to check, so
# we will delay output substantially if we only
# print one line per condition check.
# Adding a tight loop here mitigates the effect,
# ensuring that most of the output directly from
# setup.sh is sent to tee_output before the outer
# loop exits and prints things like "setup.sh exited"
#
for i in xrange(10):
try:
line = nbsr.readline(0.05)
if line:
tee_output(prefix, line)
# Reset Travis-CI timer
travis_timeout = datetime.now() + timedelta(minutes = 5)
except setup_util.EndOfStream:
tee_output(prefix, "Setup has terminated\n")
break
time_remaining = timeout - datetime.now()
if (travis_timeout - datetime.now()).total_seconds() < 0:
sys.stdout.write(prefix + 'Printing so Travis-CI does not time out\n')
sys.stdout.write(prefix + "Status: Poll: %s, Port %s bound: %s, Time Left: %s\n" % (
p.poll(), self.port, self.benchmarker.is_port_bound(self.port), time_remaining))
sys.stdout.flush()
travis_timeout = datetime.now() + timedelta(minutes = 5)
# Did we time out?
if time_remaining.total_seconds() < 0:
tee_output(prefix, "%s.sh timed out!! Aborting...\n" % self.setup_file)
p.kill()
return 1
# What's our return code?
# If setup.sh has terminated, use that code
# Otherwise, detect if the port was bound
tee_output(prefix, "Status: Poll: %s, Port %s bound: %s, Time Left: %s\n" % (
p.poll(), self.port, self.benchmarker.is_port_bound(self.port), time_remaining))
retcode = (p.poll() if p.poll() is not None else 0 if self.benchmarker.is_port_bound(self.port) else 1)
if p.poll() is not None:
tee_output(prefix, "%s.sh process exited naturally with %s\n" % (self.setup_file, p.poll()))
elif self.benchmarker.is_port_bound(self.port):
tee_output(prefix, "Bound port detected on %s\n" % self.port)
# Before we return control to the benchmarker, spin up a
# thread to keep an eye on the pipes in case the running
# framework uses stdout/stderr. Once all processes accessing
# the subprocess.PIPEs are dead, this thread will terminate.
# Use a different prefix to indicate this is the framework
# speaking
prefix = "Server %s: " % self.name
def watch_child_pipes(nbsr, prefix):
while True:
try:
line = nbsr.readline(60)
if line:
tee_output(prefix, line)
except setup_util.EndOfStream:
tee_output(prefix, "Framework processes have terminated\n")
return
watch_thread = Thread(target = watch_child_pipes,
args = (nbsr, prefix))
watch_thread.daemon = True
watch_thread.start()
logging.info("Executed %s.sh, returning %s", self.setup_file, retcode)
os.chdir(previousDir)
return retcode