from subprocess import Popen, PIPE
from time import sleep
import imp
import os
import re
import sys
POLYORB_CONF = "POLYORB_CONF"
RLIMIT = int(os.environ['RLIMIT'])
TEST_NAME = os.environ['TEST_NAME']
# Restore testsuite environment
Env().restore(os.environ['TEST_CONFIG'])
# If POLYORB_TEST_VERBOSE is set to true, then output more data
VERBOSE = Env().options.verbose # set by testsuite.py
# Main testsuite source dir
SRC_DIR = Env().options.testsuite_src_dir
# All executable tests path are relative to PolyORB testsuite build dir
BASE_DIR = os.path.join(Env().options.build_dir, 'testsuite')
# Import config module, which is generated by configure in the testsuite
# build directory (so not on the Python search path).
config = imp.load_source('config', os.path.join(BASE_DIR, 'tests',
'config.py'))
def main():
"""Run the testsuite"""
m = Main()
add_mainloop_options(m, extended_options=True)
add_run_test_options(m)
m.add_option("--diffs",
dest="view_diffs",
action="store_true",
default=False,
help="show diffs on stdout")
m.parse_args()
# Various files needed or created by the testsuite
# creates :
# the ouput directory (out by default)
# the report file
# the results file
setup_result_dir(m.options)
if m.args:
test_list = [t.strip('/') for t in m.args]
else:
test_list = sorted(glob('tests/*'))
env = Env()
# add support module path
python_lib = os.path.join(os.getcwd(), 'lib', 'python')
Env().add_search_path("PYTHONPATH", python_lib)
env.add_search_path('PYTHONPATH', os.getcwd())
discs = [env.target.platform]
if m.options.discs:
discs += m.options.discs.split(',')
test_metrics = {'total': len(test_list), 'uok': 0}
# Generate a standard 'collect_result' function...
generated_collect_result = generate_collect_result(
result_dir=m.options.output_dir,
results_file=m.options.results_file,
output_diff=m.options.view_diffs,
metrics=test_metrics)
# ... and then wrap that generated 'collect_result' function in something
# that will also accumulate 'UOK' test results
def collect_result_including_uok(name, process, _job_info):
generated_collect_result(name, process, _job_info)
test_name = os.path.basename(name)
test_result = split_file(m.options.output_dir + '/' + test_name +
'.result',
ignore_errors=True)
if test_result:
test_status = test_result[0].split(':')[0]
if test_status == 'UOK':
test_metrics['uok'] += 1
run_testcase = generate_run_testcase('run-test', discs, m.options)
MainLoop(test_list, run_testcase, collect_result_including_uok,
m.options.mainloop_jobs)
print "Summary: Ran %(run)s/%(total)s tests, with %(failed)s failed, %(crashed)s crashed, %(uok)s unexpectedly passed." % test_metrics
# Generate the report file
ReportDiff(m.options.output_dir,
m.options.old_output_dir).txt_image(m.options.report_file)
if test_metrics['failed'] > 0 or test_metrics[
'crashed'] > 0 or test_metrics['uok'] > 0:
sys.exit(1)
"""
This module contains support functions for all test.py
"""
import logging
import os
import sys
from glob import glob
from gnatpython.env import Env
from gnatpython.fileutils import cd, mv
from gnatpython.ex import Run
Env().restore(os.environ['TEST_CONFIG'])
# Move to test directory
ROOT_DIR = os.getcwd()
TEST_DIR = os.path.dirname(sys.modules['__main__'].__file__)
TEST_NAME = os.path.basename(TEST_DIR)
def setup():
cd(TEST_DIR)
for prj in glob('*.gpr'):
with open(prj) as prj_orig:
lines = [line for line in prj_orig]
with open(prj + '.new', 'w') as prj_new:
for line in lines:
line = line.replace('../common',
os.path.join(ROOT_DIR, 'common'))
prj_new.write(line)
def run_testsuite(test_driver):
"""Run the testsuite
PARAMETERS
test_driver: path to the test driver (e.g. lib/python/run-test)
"""
options = __parse_options()
env = Env()
if options.vc_timeout:
os.environ["vc_timeout"] = str(options.vc_timeout)
if options.debug:
os.environ["debug"] = "true"
if options.verbose:
os.environ["verbose"] = "true"
if options.inverse_prover:
os.environ["inverse_prover"] = "true"
if options.benchmarks:
os.environ["benchmarks"] = "true"
if options.cache:
os.environ["cache"] = "true"
if options.coverage:
os.environ["coverage"] = "true"
if options.z3_counterexample:
os.environ["z3_counterexample"] = "true"
if options.test_list:
with open(options.test_list, 'r') as f:
test_list = f.readlines()
test_list =\
map(lambda s: os.path.join("tests", s.strip()), test_list)
test_list = [t for t in test_list if os.path.isdir(t)]
elif options.exact_name:
test_name = os.path.join('tests/', options.run_test)
if os.path.isdir(test_name):
test_list = [test_name]
else:
print 'error: test \'' + options.run_test + '\' not found'
exit(1)
elif options.pattern:
test_list = filter_list('tests/*')
reg = re.compile(options.pattern)
test_list = [
test for test in test_list if test_contains_pattern(test, reg)
]
else:
test_list = [
t for t in filter_list('tests/*', options.run_test)
if os.path.isdir(t)
]
# Various files needed or created by the testsuite
setup_result_dir(options)
discs = env.discriminants
if options.discs:
discs += options.discs
run_testcase = generate_run_testcase(test_driver, discs, options)
collect_result = generate_collect_result(options.output_dir,
options.results_file,
options.view_diffs)
MainLoop(test_list, run_testcase, collect_result, options.mainloop_jobs)
# Write report
with open(options.output_dir + '/discs', 'w') as discs_f:
discs_f.write(" ".join(discs))
ReportDiff(options.output_dir,
options.old_output_dir).txt_image(options.report_file)
from SUITE import control
from SUITE.control import GPRCLEAN, BUILDER, LANGINFO
from SUITE.cutils import indent_after_first_line, lines_of, ndirs_in
# This module is loaded as part of a Run operation for a test.py
# file found and launched by the toplevel driver
# This is where the toplevel invocation was issued:
ROOT_DIR = os.getcwd()
# And this is the relative directory where test.py was found:
TEST_DIR = os.path.dirname(sys.modules['__main__'].__file__)
env = Env()
# ==================
# == ReportOutput ==
# ==================
# Internal helper to dispatch information to test.py.err/log/out
class _ReportOutput(object):
"""A class that allows us to write some text to a report file, while
bufferizing part of it until we know whether this part should also
be printed on standard output or not. The idea is to buffer the
output generated for each driver until the end of the test, and then
print that output to stdout if we then determine that the test failed.
from gnatpython.env import Env
from test_support import Run, spark_install_path
import os.path
installdir = spark_install_path()
bindir = os.path.join(installdir, 'libexec', 'spark', 'bin')
Env().add_path(bindir)
process = Run(["cvc4", "--show-config"])
lines = process.out.splitlines()
# first three lines of cvc4 output contain date and exact compiler version, so
# remove this output. We also remove the "scm" line which refers to the exact
# git commit in some builds.
for line in lines[3:]:
if not line.startswith("scm"):
print line
def main():
"""Run the testsuite and generate reports"""
# Parse the command lines options
m = Main(add_targets_options=True)
add_mainloop_options(m)
add_run_test_options(m)
m.add_option('--diffs',
dest='diffs',
action='store_true',
default=False,
help='show diffs on stdout')
m.add_option("--old-result-dir",
type="string",
default=None,
help="Old result dir (to generate the report)")
m.add_option('-b',
'--build-dir',
dest='build_dir',
help='separate PolyORB build directory')
m.add_option('--testsuite-src-dir',
dest='testsuite_src_dir',
help='path to polyorb testsuite sources')
m.add_option('--coverage',
dest='coverage',
action='store_true',
default=False,
help='generate coverage information')
m.parse_args()
# Various files needed or created by the testsuite
results_file = m.options.output_dir + '/results'
report_file = m.options.output_dir + '/report'
if not m.options.failed_only:
rm(m.options.output_dir, True)
mkdir(m.options.output_dir)
# Add current directory in PYTHONPATH (to find test_utils.py)
env = Env()
env.add_search_path('PYTHONPATH', os.path.join(os.getcwd(), 'tests'))
fixed_support_dir = os.path.join(os.getcwd(), 'fixed_support_dir')
env.add_search_path('FIXED_SUPPORT_DIR', fixed_support_dir)
env.add_path(os.path.join(fixed_support_dir))
env.add_path('.') # many tests expect '.' in the PATH
# Avoid extra debug traces
os.environ['POLYORB_LOG_DEFAULT'] = 'error'
# Generate the discs list for test.opt parsing
# Always add 'ALL'
common_discs = Env().discriminants
# Be backward compatible with the old IDL tests
# Set the polyorb discriminant and export the IDLCOMP
# environment variable.
common_discs.append('PolyORB')
common_discs.append('PolyORB_IAC')
os.environ['IDLCOMP'] = 'iac'
# Retrieve also the polyorb specific discriminants
p = Run([
which('bash'),
which('polyorb-config').replace('\\', '/'), '--config'
])
# First find the support application perso.
match = re.search('Application *personalities *: (.+)', p.out)
if match is not None:
common_discs += ['app_%s' % k for k in match.group(1).split()]
# Then the supported protocols
match = re.search('Protocol *personalities *: (.+)', p.out)
if match is not None:
common_discs += ['proto_%s' % k for k in match.group(1).split()]
# Then the supported services
match = re.search('Services *: (.+)', p.out)
if match is not None:
common_discs += ['serv_%s' % k for k in match.group(1).split()]
# Do we have ssl support ?
if re.search('SSL *support *: *yes', p.out):
common_discs.append('ssl_support')
with open(m.options.output_dir + '/discs', 'w') as f_disk:
f_disk.write(", ".join(common_discs))
# Expand ~ and ~user contructions for user PATH
if m.options.build_dir is None:
m.options.build_dir = os.path.join(os.getcwd(), os.pardir)
else:
m.options.build_dir = os.path.expanduser(m.options.build_dir)
if m.options.testsuite_src_dir is None:
m.options.testsuite_src_dir = os.path.join(os.getcwd())
else:
m.options.testsuite_src_dir = os.path.expanduser(
m.options.testsuite_src_dir)
# Compute the test list
if m.args:
test_glob = m.args[0]
else:
test_glob = None
test_list = filter_list('./tests/*/*/*/test.py', test_glob)
if os.path.isdir('regtests'):
test_list.extend(filter_list('./regtests/*/test.*', test_glob))
collect_result = generate_collect_result(m.options.output_dir,
results_file, m.options.diffs)
run_testcase = generate_run_testcase('tests/run-test.py', common_discs,
m.options)
os.environ['TEST_CONFIG'] = os.path.join(os.getcwd(), 'env.dump')
env.options = m.options
env.log_dir = os.path.join(os.getcwd(), 'log')
env.store(os.environ['TEST_CONFIG'])
if len(test_list) == 0:
logger.error("No matching test found")
return
MainLoop(test_list, run_testcase, collect_result, m.options.mainloop_jobs)
# Generate the report file
ReportDiff(m.options.output_dir,
m.options.old_result_dir).txt_image(report_file)
def __init__(self,
test,
discs,
result_dir,
temp_dir=Env().tmp_dir,
enable_cleanup=True,
restricted_discs=None,
test_args=None,
failed_only=False,
default_timeout=780,
use_basename=True):
"""TestRunner constructor.
:param test: location of the test
:type test: str
:param discs: list of discriminants
:type discs: list[str]
:param result_dir: directory in which results will be stored
:type result_dir: str
:param temp_dir: temporary directory used during test run
:type temp_dir: str
:param enable_cleanup: whether the temporary files needs to be removed
:type enable_cleanup: bool
:param restricted_discs: None or a list of discriminants
:type restricted_discs: list[str] | None
:param test_args: ???
:param failed_only: run failed only
:type failed_only: bool
:param default_timeout: timeout when executing a test
:type default_timeout: int
:param use_basename: if True use the test basename to get the test name
else use the relative path
:type use_basename: bool
"""
self.test = test.rstrip('/')
self.discs = discs
self.cmd_line = None
self.test_args = test_args
self.enable_cleanup = enable_cleanup
self.restricted_discs = restricted_discs
self.skip = False # if True, do not run execute()
# Test name
if use_basename:
self.test_name = os.path.basename(self.test)
else:
self.test_name = os.path.relpath(self.test, os.getcwd())
# Prefix of files holding the test result
self.result_prefix = result_dir + '/' + self.test_name
mkdir(os.path.dirname(self.result_prefix))
# Temp directory in which the test will be run
self.work_dir = os.path.realpath(
os.path.join(temp_dir,
'tmp-test-%s-%d' % (self.test_name, os.getpid())))
self.output = self.work_dir + '/tmpout'
self.output_filtered = self.work_dir + '/tmpout.filtered'
self.diff_output = self.work_dir + '/diff'
self.cmdlog = self.work_dir + '/' + self.test_name + '.log'
# Initial test status
self.result = {'result': 'UNKNOWN', 'msg': '', 'is_failure': True}
# Some tests save the pids of spawned background processes in
# work_dir/.pids. The TEST_WORK_DIR environment variable is used to
# pass the working directory location.
os.environ['TEST_WORK_DIR'] = self.work_dir
if failed_only:
# Read old result now
previous_result = self.read_result()
if previous_result in IS_STATUS_FAILURE \
and not IS_STATUS_FAILURE[previous_result]:
# We don't need to run this test. Return now
self.skip = True
return
# Be sure to be a sane environment
rm(self.result_prefix + '.result')
rm(self.result_prefix + '.out')
rm(self.result_prefix + '.expected')
rm(self.result_prefix + '.diff')
rm(self.result_prefix + '.log')
rm(self.result_prefix + '.out.filtered')
# Initialize options defaults (can be modified with test.opt).
# By default a test is not DEAD, SKIP nor XFAIL. Its maximum execution
# time is 780s. Test script is test.cmd and output is compared against
# test.out.
self.opt_results = {
'RLIMIT': str(default_timeout),
'DEAD': None,
'XFAIL': False,
'SKIP': None,
'OUT': 'test.out',
'CMD': 'test.cmd',
'FILESIZE_LIMIT': None,
'TIMING': None,
'NOTE': None
}
self.opt_file = 'test.opt'
# test.cmd have priority, if not found use test.py
if not os.path.isfile(self.test +
'/test.cmd') and os.path.isfile(self.test +
'/test.py'):
self.opt_results['CMD'] = 'test.py'
def add_run_test_options(m):
"""Add standard test driver options."""
run_test_opts = m.create_option_group("Test driver options")
run_test_opts.add_option("-o",
"--output-dir",
dest="output_dir",
metavar="DIR",
default="./out",
help="select output dir")
run_test_opts.add_option("--timeout",
default='780',
metavar="SECONDS",
help="Default timeout")
run_test_opts.add_option("-d",
"--discriminants",
dest="discs",
metavar="DISCS",
default="ALL",
help="set discriminants")
run_test_opts.add_option("-t",
"--temp-dir",
dest="tmp",
metavar="DIR",
default=Env().tmp_dir)
run_test_opts.add_option("-e",
"--env-file",
dest="env_file",
metavar="FILE",
default="load env file")
run_test_opts.add_option("--disable-cleanup",
dest="enable_cleanup",
action="store_false",
default=True,
help="disable cleanup of working space")
run_test_opts.add_option(
"--dump-environ",
dest="dump_environ",
action="store_true",
default=False,
help="Dump all environment variables in a file named environ.sh,"
" located in the output directory (see --output-dir). This"
" file can then be sourced from a Bourne shell to recreate"
" the environement that existed when this testsuite was run"
" to produce a given testsuite report.")
run_test_opts.add_option("-r",
"--restricted-mode",
dest="restricted_discs",
metavar="DISCS",
default=None,
help="enable restricted mode")
run_test_opts.add_option(
'-f',
'--failed-only',
action="store_true",
help="run failed only - skip the test is last result is OK")
run_test_opts.add_option(
'--use-basename',
action='store_true',
help="Use os.path.basename to get the real name of a test. "
"Note that this will only work if you don't have two tests with "
"the same name in your test directories")
m.add_option_group(run_test_opts)
def compute_cmd_line_cmd(self, filesize_limit):
"""Compute self.cmd_line and preprocess the test script.
This function is called by compute_cmd_line
"""
cmd = self.opt_results['CMD']
if Env().host.os.name != 'windows':
script = split_file(cmd)
# The test is run on a Unix system but has a 'cmd' syntax.
# Convert it to Bourne shell syntax.
cmdfilter = Filter()
cmdfilter.append([r'-o(.*).exe', r'-o \1'])
cmdfilter.append([r'%([^ ]*)%', r'"$\1"'])
cmdfilter.append([r'(\032|\015)', r''])
cmdfilter.append(
[r'set *([^ =]+) *= *([^ ]*)', r'\1="\2"; export \1'])
script = cmdfilter.process(script)
cmd = self.work_dir + '/__test.sh'
echo_to_file(cmd, 'PATH=.:$PATH; export PATH\n')
# Compute effective file size limit on Unix system.
if filesize_limit > 0:
# File size limit can be specified either by a default or by
# mean of FILESIZE_LIMIT command in the test test.opt. When
# both are specified use the upper limit (note that 0 means
# unlimited).
opt_limit = self.opt_results['FILESIZE_LIMIT']
if opt_limit is not None:
try:
opt_limit = int(opt_limit)
except TypeError:
opt_limit = filesize_limit
else:
opt_limit = filesize_limit
if opt_limit != 0:
if filesize_limit < opt_limit:
filesize_limit = opt_limit
# Limit filesize. Argument to ulimit is a number of blocks
# (512 bytes) so multiply by two the argument given by the
# user. Filesize limit is not supported on Windows.
echo_to_file(cmd, 'ulimit -f %s\n' % (filesize_limit * 2),
True)
# Source support.sh in TEST_SUPPORT_DIR if set
if 'TEST_SUPPORT_DIR' in os.environ and os.path.isfile(
os.environ['TEST_SUPPORT_DIR'] + '/support.sh'):
echo_to_file(cmd, '. $TEST_SUPPORT_DIR/support.sh\n', True)
echo_to_file(cmd, script, True)
self.cmd_line += ['bash', cmd]
else:
# On windows system, use cmd to run the script.
if cmd[-4:] != '.cmd':
# We are about to use cmd.exe to run a test. In this case,
# ensure that the file extension is .cmd otherwise a dialog box
# will popup asking to choose the program that should be used
# to run the script.
cp(cmd, self.work_dir + '/test__.cmd')
cmd = self.work_dir + '/test__.cmd'
self.cmd_line += ['cmd.exe', '/q', '/c', cmd]
