def run_test(test,
status_queue,
kill_switch,
verbose=False,
stuck_timeout=None,
grace_period=0):
# Construct the base command.
command = ["bash", "-c", test.command]
# If we have a "pidspace" program, use that to ensure that programs
# that double-fork can't continue running after the parent command
# dies.
if which("pidspace") is not None:
command = [which("pidspace"), "--"] + command
# Print command and path.
if verbose:
print("\n")
if os.path.abspath(test.cwd) != os.path.abspath(os.getcwd()):
path = " [%s]" % os.path.relpath(test.cwd)
else:
path = ""
print(" command: %s%s" % (test.command, path))
# Determine where stdout should go. We can't print it live to stdout and
# also capture it, unfortunately.
output = sys.stdout if verbose else subprocess.PIPE
# Start timing.
start_time = datetime.datetime.now()
# Start the command.
peak_mem_usage = None
try:
process = subprocess.Popen(command,
stdout=output,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
cwd=test.cwd)
except:
output = "Exception while running test:\n\n%s" % (
traceback.format_exc())
if verbose:
print(output)
status_queue.put({
'name': test.name,
'status': ERROR,
'output': output,
'real_time': datetime.datetime.now() - start_time,
'cpu_time': 0,
'mem_usage': peak_mem_usage
})
return
# Now running the test.
# Wrap in a list to prevent nested functions getting the wrong scope
test_status = [RUNNING]
# If we exit for some reason, attempt to kill our test processes.
def emergency_stop():
if test_status[0] is RUNNING:
kill_family(grace_period, process.pid)
atexit.register(emergency_stop)
# Setup a timer for the timeout.
def do_timeout():
if test_status[0] is RUNNING:
test_status[0] = TIMEOUT
kill_family(grace_period, process.pid)
timer = threading.Timer(test.timeout, do_timeout)
if test.timeout > 0:
timer.start()
# Poll the kill switch.
def watch_kill_switch():
while True:
interval = 1.0
if test_status[0] is not RUNNING:
break
if kill_switch.wait(1):
if test_status[0] is not RUNNING:
break
test_status[0] = CANCELLED
kill_family(grace_period, process.pid)
time.sleep(interval)
kill_switch_thread = threading.Thread(target=watch_kill_switch)
kill_switch_thread.daemon = True
kill_switch_thread.start()
with cpuusage.process_poller(process.pid) as c:
# Inactivity timeout
low_cpu_usage = 0.05 # 5%
cpu_history = collections.deque() # sliding window
cpu_usage_total = [0] # workaround for variable scope
# Also set a CPU timeout. We poll the cpu usage periodically.
def cpu_timeout():
last_cpu_usage = 0
interval = min(0.5, test.cpu_timeout / 10.0)
while test_status[0] is RUNNING:
thread_cpu_usage = c.cpu_usage()
if stuck_timeout:
# append to window
now = time.time()
if not cpu_history:
cpu_history.append(
(time.time(), thread_cpu_usage / interval))
else:
real_interval = now - cpu_history[-1][0]
cpu_increment = thread_cpu_usage - last_cpu_usage
cpu_history.append(
(now, cpu_increment / real_interval))
cpu_usage_total[0] += cpu_history[-1][1]
# pop from window, ensuring that window covers at least stuck_timeout interval
while len(cpu_history) > 1 and cpu_history[1][
0] + stuck_timeout <= now:
cpu_usage_total[0] -= cpu_history[0][1]
cpu_history.popleft()
if (now - cpu_history[0][0] >= stuck_timeout
and cpu_usage_total[0] / len(cpu_history) <
low_cpu_usage):
test_status[0] = STUCK
kill_family(grace_period, process.pid)
break
if thread_cpu_usage > test.cpu_timeout:
test_status[0] = CPU_TIMEOUT
kill_family(grace_period, process.pid)
break
last_cpu_usage = thread_cpu_usage
time.sleep(interval)
cpu_timer = None
if test.cpu_timeout > 0:
cpu_timer = threading.Thread(target=cpu_timeout)
cpu_timer.daemon = True
cpu_timer.start()
with memusage.process_poller(process.pid) as m:
# Wait for the command to finish.
(output, _) = process.communicate()
peak_mem_usage = m.peak_mem_usage()
cpu_usage = c.cpu_usage()
if process.returncode == 0:
test_status[0] = PASSED
elif test_status[0] is RUNNING:
# No special status, so assume it failed by itself
test_status[0] = FAILED
if cpu_timer is not None:
# prevent cpu_timer using c after it goes away
cpu_timer.join()
# Cancel the timer. Small race here (if the timer fires just after the
# process finished), but the return code of our process should still be 0,
# and hence we won't interpret the result as a timeout.
if test_status[0] is not TIMEOUT:
timer.cancel()
if output is None:
output = ""
output = output.decode(encoding='utf8', errors='replace')
if test_status[0] in [STUCK, TIMEOUT, CPU_TIMEOUT]:
output = output + extra_timeout_output(test.name)
status_queue.put({
'name': test.name,
'status': test_status[0],
'output': output,
'real_time': datetime.datetime.now() - start_time,
'cpu_time': cpu_usage,
'mem_usage': peak_mem_usage
})
def run_test(test, verbose=False):
# Construct the base command.
command = ["bash", "-c", test.command]
# If we have a "pidspace" program, use that to ensure that programs
# that double-fork can't continue running after the parent command
# dies.
if which("pidspace") != None:
command = [which("pidspace"), "--"] + command
# Print command and path.
if verbose:
print("\n")
if os.path.abspath(test.cwd) != os.path.abspath(os.getcwd()):
path = " [%s]" % os.path.relpath(test.cwd)
else:
path = ""
print(" command: %s%s" % (test.command, path))
output = subprocess.PIPE
# Start timing.
start_time = datetime.datetime.now()
# Start the command.
peak_mem_usage = None
try:
process = subprocess.Popen(command,
stdout=output,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
cwd=test.cwd,
text=True)
lines_iter = iter(process.stdout.readline, "") # pytype: disable=attribute-error
if verbose:
for line in lines_iter:
print(line, end='')
except:
output = "Exception while running test:\n\n%s" % (
traceback.format_exc())
atexit.register(lambda: kill_family(process.pid))
if verbose:
print(output)
return (False, "ERROR", output, datetime.datetime.now() - start_time,
peak_mem_usage)
# If our program exits for some reason, attempt to kill the process.
atexit.register(lambda: kill_family(process.pid))
# Setup an alarm at the timeout.
was_timeout = [False]
def alarm_handler(sig, _):
was_timeout[0] = True
kill_family(process.pid)
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(test.timeout)
# Wait for the command to finish.
with memusage.process_poller(process.pid) as m:
(output, _) = process.communicate()
peak_mem_usage = m.peak_mem_usage()
# Cancel the alarm. Small race here (if the alarm fires just after the
# process finished), but the returncode of our process should still be 0,
# and hence we won't interpret the result as a timeout.
signal.alarm(0)
if output == None:
output = ""
if process.returncode == 0:
status = "pass"
elif was_timeout[0]:
status = "TIMEOUT"
else:
status = "FAILED"
return (process.returncode == 0, status, output,
datetime.datetime.now() - start_time, peak_mem_usage)
def run_test(test, status_queue, kill_switch, verbose=False, stuck_timeout=None, grace_period=0):
# Construct the base command.
command = ["bash", "-c", test.command]
# If we have a "pidspace" program, use that to ensure that programs
# that double-fork can't continue running after the parent command
# dies.
if which("pidspace") != None:
command = [which("pidspace"), "--"] + command
# Print command and path.
if verbose:
print("\n")
if os.path.abspath(test.cwd) != os.path.abspath(os.getcwd()):
path = " [%s]" % os.path.relpath(test.cwd)
else:
path = ""
print(" command: %s%s" % (test.command, path))
# Determine where stdout should go. We can't print it live to stdout and
# also capture it, unfortunately.
output = sys.stdout if verbose else subprocess.PIPE
# Start timing.
start_time = datetime.datetime.now()
# Start the command.
peak_mem_usage = None
cpu_usage = None
try:
process = subprocess.Popen(command,
stdout=output, stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
cwd=test.cwd)
except:
output = "Exception while running test:\n\n%s" % (traceback.format_exc())
if verbose:
print(output)
status_queue.put({'name': test.name,
'status': ERROR,
'output': output,
'real_time': datetime.datetime.now() - start_time,
'cpu_time': 0,
'mem_usage': peak_mem_usage})
return
# Now running the test.
# Wrap in a list to prevent nested functions getting the wrong scope
test_status = [RUNNING]
# If we exit for some reason, attempt to kill our test processes.
def emergency_stop():
if test_status[0] is RUNNING:
kill_family(grace_period, process.pid)
atexit.register(emergency_stop)
# Setup a timer for the timeout.
def do_timeout():
if test_status[0] is RUNNING:
test_status[0] = TIMEOUT
kill_family(grace_period, process.pid)
timer = threading.Timer(test.timeout, do_timeout)
if test.timeout > 0:
timer.start()
# Poll the kill switch.
def watch_kill_switch():
while True:
if test_status[0] is not RUNNING:
break
if kill_switch.wait(1):
if test_status[0] is not RUNNING:
break
test_status[0] = CANCELLED
kill_family(grace_period, process.pid)
kill_switch_thread = threading.Thread(target=watch_kill_switch)
kill_switch_thread.daemon = True
kill_switch_thread.start()
with cpuusage.process_poller(process.pid) as c:
# Inactivity timeout
low_cpu_usage = 0.05 # 5% -- FIXME: hardcoded
cpu_history = collections.deque() # sliding window
last_cpu_usage = 0
cpu_usage_total = [0] # workaround for variable scope
# Also set a CPU timeout. We poll the cpu usage periodically.
def cpu_timeout():
interval = min(0.5, test.cpu_timeout / 10.0)
while test_status[0] is RUNNING:
cpu_usage = c.cpu_usage()
if stuck_timeout:
# append to window
now = time.time()
if not cpu_history:
cpu_history.append((time.time(), cpu_usage / interval))
else:
real_interval = now - cpu_history[-1][0]
cpu_increment = cpu_usage - last_cpu_usage
cpu_history.append((now, cpu_increment / real_interval))
cpu_usage_total[0] += cpu_history[-1][1]
# pop from window, ensuring that window covers at least stuck_timeout interval
while len(cpu_history) > 1 and cpu_history[1][0] + stuck_timeout <= now:
cpu_usage_total[0] -= cpu_history[0][1]
cpu_history.popleft()
if (now - cpu_history[0][0] >= stuck_timeout and
cpu_usage_total[0] / len(cpu_history) < low_cpu_usage):
test_status[0] = STUCK
kill_family(grace_period, process.pid)
break
if cpu_usage > test.cpu_timeout:
test_status[0] = CPU_TIMEOUT
kill_family(grace_period, process.pid)
break
last_cpu_usage = cpu_usage
time.sleep(interval)
if test.cpu_timeout > 0:
cpu_timer = threading.Thread(target=cpu_timeout)
cpu_timer.daemon = True
cpu_timer.start()
with memusage.process_poller(process.pid) as m:
# Wait for the command to finish.
(output, _) = process.communicate()
peak_mem_usage = m.peak_mem_usage()
cpu_usage = c.cpu_usage()
if process.returncode == 0:
test_status[0] = PASSED
elif test_status[0] is RUNNING:
# No special status, so assume it failed by itself
test_status[0] = FAILED
if test.cpu_timeout > 0:
# prevent cpu_timer using c after it goes away
cpu_timer.join()
# Cancel the timer. Small race here (if the timer fires just after the
# process finished), but the returncode of our process should still be 0,
# and hence we won't interpret the result as a timeout.
if test_status[0] is not TIMEOUT:
timer.cancel()
if output == None:
output = ""
output = output.decode(encoding='utf8', errors='replace')
status_queue.put({'name': test.name,
'status': test_status[0],
'output': output,
'real_time': datetime.datetime.now() - start_time,
'cpu_time': cpu_usage,
'mem_usage': peak_mem_usage})
def run_test(test, status_queue, verbose=False):
# Construct the base command.
command = ["bash", "-c", test.command]
# If we have a "pidspace" program, use that to ensure that programs
# that double-fork can't continue running after the parent command
# dies.
if which("pidspace") != None:
command = [which("pidspace"), "--"] + command
# Print command and path.
if verbose:
print("\n")
if os.path.abspath(test.cwd) != os.path.abspath(os.getcwd()):
path = " [%s]" % os.path.relpath(test.cwd)
else:
path = ""
print(" command: %s%s" % (test.command, path))
# Determine where stdout should go. We can't print it live to stdout and
# also capture it, unfortunately.
output = sys.stdout if verbose else subprocess.PIPE
# Start timing.
start_time = datetime.datetime.now()
# Start the command.
peak_mem_usage = None
try:
process = subprocess.Popen(command,
stdout=output,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
cwd=test.cwd)
except:
output = "Exception while running test:\n\n%s" % (
traceback.format_exc())
if verbose:
print(output)
return (False, "ERROR", output, datetime.datetime.now() - start_time,
peak_mem_usage)
# If we exit for some reason, attempt to kill our test processes.
process_running = True
def emergency_stop():
if process_running:
kill_family(process.pid)
atexit.register(emergency_stop)
# Setup an alarm at the timeout.
was_timeout = [
False
] # Wrap in list to prevent do_timeout getting the wrong variable scope
def do_timeout():
was_timeout[0] = True
kill_family(process.pid)
timer = threading.Timer(test.timeout, do_timeout)
if test.timeout > 0:
timer.start()
# Wait for the command to finish.
with memusage.process_poller(process.pid) as m:
(output, _) = process.communicate()
peak_mem_usage = m.peak_mem_usage()
process_running = False
# Cancel the alarm. Small race here (if the timer fires just after the
# process finished), but the returncode of our process should still be 0,
# and hence we won't interpret the result as a timeout.
if not was_timeout[0]:
timer.cancel()
if output == None:
output = ""
if process.returncode == 0:
status = "pass"
elif was_timeout[0]:
status = "TIMEOUT"
else:
status = "FAILED"
status_queue.put({
'name': test.name,
'status': status,
'output': output,
'real_time': datetime.datetime.now() - start_time,
'mem_usage': peak_mem_usage
})
def run_test(test, verbose=False):
# Construct the base command.
command = ["bash", "-c", test.command]
# If we have a "pidspace" program, use that to ensure that programs
# that double-fork can't continue running after the parent command
# dies.
if which("pidspace") != None:
command = [which("pidspace"), "--"] + command
# Print command and path.
if verbose:
print("\n")
if os.path.abspath(test.cwd) != os.path.abspath(os.getcwd()):
path = " [%s]" % os.path.relpath(test.cwd)
else:
path = ""
print(" command: %s%s" % (test.command, path))
# Determine where stdout should go. We can't print it live to stdout and
# also capture it, unfortunately.
output = sys.stdout if verbose else subprocess.PIPE
# Start timing.
start_time = datetime.datetime.now()
# Start the command.
peak_mem_usage = None
try:
process = subprocess.Popen(command,
stdout=output, stderr=subprocess.STDOUT, stdin=subprocess.PIPE,
cwd=test.cwd)
except:
output = "Exception while running test:\n\n%s" % (traceback.format_exc())
if verbose:
print(output)
return (False, "ERROR", output, datetime.datetime.now() - start_time, peak_mem_usage)
# If our program exits for some reason, attempt to kill the process.
atexit.register(lambda: kill_family(process.pid))
# Setup an alarm at the timeout.
was_timeout = [False]
def alarm_handler(sig, _):
was_timeout[0] = True
kill_family(process.pid)
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(test.timeout)
# Wait for the command to finish.
with memusage.process_poller(process.pid) as m:
(output, _) = process.communicate()
peak_mem_usage = m.peak_mem_usage()
# Cancel the alarm. Small race here (if the alarm fires just after the
# process finished), but the returncode of our process should still be 0,
# and hence we won't interpret the result as a timeout.
signal.alarm(0)
if output == None:
output = ""
if process.returncode == 0:
status = "pass"
elif was_timeout[0]:
status = "TIMEOUT"
else:
status = "FAILED"
return (process.returncode == 0, status, output, datetime.datetime.now() - start_time, peak_mem_usage)