def _parse_lit_output(self, path, data, only_test=False):
LIT_METRIC_TO_LNT = {'compile_time': 'compile', 'exec_time': 'exec'}
# We don't use the test info, currently.
test_info = {}
test_samples = []
# FIXME: Populate with keys not to upload
ignore = self.opts.exclude_stat_from_submission
if only_test:
ignore.append('compile')
for test_data in data['tests']:
raw_name = test_data['name'].split(' :: ', 1)[1]
name = 'nts.' + raw_name.rsplit('.test', 1)[0]
is_pass = self._is_pass_code(test_data['code'])
if 'metrics' in test_data:
for k, v in test_data['metrics'].items():
if k not in LIT_METRIC_TO_LNT or LIT_METRIC_TO_LNT[
k] in ignore:
continue
test_samples.append(
lnt.testing.TestSamples(
name + '.' + LIT_METRIC_TO_LNT[k], [v], test_info))
if self._test_failed_to_compile(raw_name, path):
test_samples.append(
lnt.testing.TestSamples(name + '.compile.status',
[lnt.testing.FAIL], test_info))
elif not is_pass:
test_samples.append(
lnt.testing.TestSamples(
name + '.exec.status',
[self._get_lnt_code(test_data['code'])], test_info))
# FIXME: Add more machine info!
run_info = {'tag': 'nts'}
run_info.update(self._get_cc_info())
run_info['run_order'] = run_info['inferred_run_order']
machine_info = {}
machine = lnt.testing.Machine("jm", machine_info)
run = lnt.testing.Run(self.start_time, timestamp(), info=run_info)
report = lnt.testing.Report(machine, run, test_samples)
return report
def run_test(self, name, args):
# FIXME: Add more detailed usage information
parser = OptionParser("%s [options] test-suite" % name)
group = OptionGroup(parser, "Sandbox options")
group.add_option("-S", "--sandbox", dest="sandbox_path",
help="Parent directory to build and run tests in",
type=str, default=None, metavar="PATH")
group.add_option("", "--no-timestamp", dest="timestamp_build",
action="store_false", default=True,
help="Don't timestamp build directory (for testing)")
group.add_option("", "--no-configure", dest="run_configure",
action="store_false", default=True,
help="Don't run CMake if CMakeCache.txt is present"
" (only useful with --no-timestamp")
parser.add_option_group(group)
group = OptionGroup(parser, "Inputs")
group.add_option("", "--test-suite", dest="test_suite_root",
type=str, metavar="PATH", default=None,
help="Path to the LLVM test-suite sources")
group.add_option("", "--test-externals", dest="test_suite_externals",
type=str, metavar="PATH",
help="Path to the LLVM test-suite externals")
group.add_option("", "--cmake-define", dest="cmake_defines",
action="append",
help=("Defines to pass to cmake. These do not require the "
"-D prefix and can be given multiple times. e.g.: "
"--cmake-define A=B => -DA=B"))
group.add_option("-C", "--cmake-cache", dest="cmake_cache",
help=("Use one of the test-suite's cmake configurations."
" Ex: Release, Debug"))
parser.add_option_group(group)
group = OptionGroup(parser, "Test compiler")
group.add_option("", "--cc", dest="cc", metavar="CC",
type=str, default=None,
help="Path to the C compiler to test")
group.add_option("", "--cxx", dest="cxx", metavar="CXX",
type=str, default=None,
help="Path to the C++ compiler to test (inferred from"
" --cc where possible")
group.add_option("", "--llvm-arch", dest="llvm_arch",
type='choice', default=None,
help="Override the CMake-inferred architecture",
choices=TEST_SUITE_KNOWN_ARCHITECTURES)
group.add_option("", "--cross-compiling", dest="cross_compiling",
action="store_true", default=False,
help="Inform CMake that it should be cross-compiling")
group.add_option("", "--cross-compiling-system-name", type=str,
default=None, dest="cross_compiling_system_name",
help="The parameter to pass to CMAKE_SYSTEM_NAME when"
" cross-compiling. By default this is 'Linux' "
"unless -arch is in the cflags, in which case "
"it is 'Darwin'")
group.add_option("", "--cppflags", type=str, action="append",
dest="cppflags", default=[],
help="Extra flags to pass the compiler in C or C++ mode. "
"Can be given multiple times")
group.add_option("", "--cflags", type=str, action="append",
dest="cflags", default=[],
help="Extra CFLAGS to pass to the compiler. Can be "
"given multiple times")
group.add_option("", "--cxxflags", type=str, action="append",
dest="cxxflags", default=[],
help="Extra CXXFLAGS to pass to the compiler. Can be "
"given multiple times")
parser.add_option_group(group)
group = OptionGroup(parser, "Test selection")
group.add_option("", "--test-size", type='choice', dest="test_size",
choices=['small', 'regular', 'large'], default='regular',
help="The size of test inputs to use")
group.add_option("", "--benchmarking-only",
dest="benchmarking_only", action="store_true",
default=False,
help="Benchmarking-only mode. Disable unit tests and "
"other flaky or short-running tests")
group.add_option("", "--only-test", dest="only_test", metavar="PATH",
type=str, default=None,
help="Only run tests under PATH")
parser.add_option_group(group)
group = OptionGroup(parser, "Test Execution")
group.add_option("-j", "--threads", dest="threads",
help="Number of testing (and optionally build) "
"threads", type=int, default=1, metavar="N")
group.add_option("", "--build-threads", dest="build_threads",
help="Number of compilation threads, defaults to "
"--threads", type=int, default=0, metavar="N")
group.add_option("", "--use-perf", dest="use_perf",
help=("Use Linux perf for high accuracy timing, profile "
"information or both"),
type='choice',
choices=['none', 'time', 'profile', 'all'],
default='none')
group.add_option("", "--run-under", dest="run_under",
help="Wrapper to run tests under ['%default']",
type=str, default="")
group.add_option("", "--exec-multisample", dest="exec_multisample",
help="Accumulate execution test data from multiple runs",
type=int, default=1, metavar="N")
group.add_option("", "--compile-multisample", dest="compile_multisample",
help="Accumulate compile test data from multiple runs",
type=int, default=1, metavar="N")
group.add_option("-d", "--diagnose", dest="diagnose",
help="Produce a diagnostic report for a particular "
"test, this will not run all the tests. Must be"
" used in conjunction with --only-test.",
action="store_true", default=False,)
parser.add_option_group(group)
group = OptionGroup(parser, "Output Options")
group.add_option("", "--no-auto-name", dest="auto_name",
help="Don't automatically derive submission name",
action="store_false", default=True)
group.add_option("", "--run-order", dest="run_order", metavar="STR",
help="String to use to identify and order this run",
action="store", type=str, default=None)
group.add_option("", "--submit", dest="submit_url", metavar="URLORPATH",
help=("autosubmit the test result to the given server"
" (or local instance) [%default]"),
type=str, default=None)
group.add_option("", "--commit", dest="commit",
help=("whether the autosubmit result should be committed "
"[%default]"),
type=int, default=True)
group.add_option("-v", "--verbose", dest="verbose",
help="show verbose test results",
action="store_true", default=False)
group.add_option("", "--succinct-compile-output",
help="run Make without VERBOSE=1",
action="store_true", dest="succinct")
group.add_option("", "--exclude-stat-from-submission",
dest="exclude_stat_from_submission",
help="Do not submit the stat of this type [%default]",
action='append', choices=KNOWN_SAMPLE_KEYS,
type='choice', default=[])
group.add_option("", "--single-result", dest="single_result",
help=("only execute this single test and apply "
"--single-result-predicate to calculate the "
"exit status"))
group.add_option("", "--single-result-predicate",
dest="single_result_predicate",
help=("the predicate to apply to calculate the exit "
"status (with --single-result)"),
default="status")
parser.add_option_group(group)
group = OptionGroup(parser, "Test tools")
group.add_option("", "--use-cmake", dest="cmake", metavar="PATH",
type=str, default="cmake",
help="Path to CMake [cmake]")
group.add_option("", "--use-make", dest="make", metavar="PATH",
type=str, default="make",
help="Path to Make [make]")
group.add_option("", "--use-lit", dest="lit", metavar="PATH",
type=str, default="llvm-lit",
help="Path to the LIT test runner [llvm-lit]")
(opts, args) = parser.parse_args(args)
self.opts = opts
if len(args) == 0:
self.nick = platform.uname()[1]
elif len(args) == 1:
self.nick = args[0]
else:
parser.error("Expected no positional arguments (got: %r)" % (args,))
for a in ['cross_compiling', 'cross_compiling_system_name', 'llvm_arch']:
if getattr(opts, a):
parser.error('option "%s" is not yet implemented!' % a)
if self.opts.sandbox_path is None:
parser.error('--sandbox is required')
# Option validation.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
parser.error('--cc does not point to a valid executable.')
# If there was no --cxx given, attempt to infer it from the --cc.
if opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
note("Inferred C++ compiler under test as: %r" % (opts.cxx,))
else:
parser.error("unable to infer --cxx - set it manually.")
else:
opts.cxx = resolve_command_path(opts.cxx)
if not os.path.exists(opts.cxx):
parser.error("invalid --cxx argument %r, does not exist" % (opts.cxx))
if opts.test_suite_root is None:
parser.error('--test-suite is required')
if not os.path.exists(opts.test_suite_root):
parser.error("invalid --test-suite argument, does not exist: %r" % (
opts.test_suite_root))
if opts.test_suite_externals:
if not os.path.exists(opts.test_suite_externals):
parser.error(
"invalid --test-externals argument, does not exist: %r" % (
opts.test_suite_externals,))
opts.cmake = resolve_command_path(opts.cmake)
if not isexecfile(opts.cmake):
parser.error("CMake tool not found (looked for %s)" % opts.cmake)
opts.make = resolve_command_path(opts.make)
if not isexecfile(opts.make):
parser.error("Make tool not found (looked for %s)" % opts.make)
opts.lit = resolve_command_path(opts.lit)
if not isexecfile(opts.lit):
parser.error("LIT tool not found (looked for %s)" % opts.lit)
if opts.run_under:
split = shlex.split(opts.run_under)
split[0] = resolve_command_path(split[0])
if not isexecfile(split[0]):
parser.error("Run under wrapper not found (looked for %s)" %
opts.run_under)
if opts.single_result:
# --single-result implies --only-test
opts.only_test = opts.single_result
if opts.only_test:
# --only-test can either point to a particular test or a directory.
# Therefore, test_suite_root + opts.only_test or
# test_suite_root + dirname(opts.only_test) must be a directory.
path = os.path.join(self.opts.test_suite_root, opts.only_test)
parent_path = os.path.dirname(path)
if os.path.isdir(path):
opts.only_test = (opts.only_test, None)
elif os.path.isdir(parent_path):
opts.only_test = (os.path.dirname(opts.only_test),
os.path.basename(opts.only_test))
else:
parser.error("--only-test argument not understood (must be a " +
" test or directory name)")
if opts.single_result and not opts.only_test[1]:
parser.error("--single-result must be given a single test name, not a " +
"directory name")
opts.cppflags = ' '.join(opts.cppflags)
opts.cflags = ' '.join(opts.cflags)
opts.cxxflags = ' '.join(opts.cxxflags)
if opts.diagnose:
if not opts.only_test:
parser.error("--diagnose requires --only-test")
self.start_time = timestamp()
# Work out where to put our build stuff
if self.opts.timestamp_build:
ts = self.start_time.replace(' ', '_').replace(':', '-')
build_dir_name = "test-%s" % ts
else:
build_dir_name = "build"
basedir = os.path.join(self.opts.sandbox_path, build_dir_name)
self._base_path = basedir
# We don't support compiling without testing as we can't get compile-
# time numbers from LIT without running the tests.
if opts.compile_multisample > opts.exec_multisample:
note("Increasing number of execution samples to %d" %
opts.compile_multisample)
opts.exec_multisample = opts.compile_multisample
if opts.auto_name:
# Construct the nickname from a few key parameters.
cc_info = self._get_cc_info()
cc_nick = '%s_%s' % (cc_info['cc_name'], cc_info['cc_build'])
self.nick += "__%s__%s" % (cc_nick,
cc_info['cc_target'].split('-')[0])
note('Using nickname: %r' % self.nick)
# If we are doing diagnostics, skip the usual run and do them now.
if opts.diagnose:
return self.diagnose()
# Now do the actual run.
reports = []
for i in range(max(opts.exec_multisample, opts.compile_multisample)):
c = i < opts.compile_multisample
e = i < opts.exec_multisample
reports.append(self.run(self.nick, compile=c, test=e))
report = self._create_merged_report(reports)
# Write the report out so it can be read by the submission tool.
report_path = os.path.join(self._base_path, 'report.json')
with open(report_path, 'w') as fd:
fd.write(report.render())
return self.submit(report_path, self.opts, commit=True)
def _parse_lit_output(self, path, data, only_test=False):
LIT_METRIC_TO_LNT = {
'compile_time': 'compile',
'exec_time': 'exec',
'score': 'score',
'hash': 'hash'
}
LIT_METRIC_CONV_FN = {
'compile_time': float,
'exec_time': float,
'score': float,
'hash': str
}
# We don't use the test info, currently.
test_info = {}
test_samples = []
# FIXME: Populate with keys not to upload
ignore = self.opts.exclude_stat_from_submission
if only_test:
ignore.append('compile')
profiles_to_import = []
for test_data in data['tests']:
raw_name = test_data['name'].split(' :: ', 1)[1]
name = 'nts.' + raw_name.rsplit('.test', 1)[0]
is_pass = self._is_pass_code(test_data['code'])
# If --single-result is given, exit based on --single-result-predicate
if self.opts.single_result and \
raw_name == self.opts.single_result+'.test':
env = {'status': is_pass}
if 'metrics' in test_data:
for k,v in test_data['metrics'].items():
env[k] = v
if k in LIT_METRIC_TO_LNT:
env[LIT_METRIC_TO_LNT[k]] = v
status = eval(self.opts.single_result_predicate, {}, env)
sys.exit(0 if status else 1)
if 'metrics' in test_data:
for k,v in test_data['metrics'].items():
if k == 'profile':
profiles_to_import.append( (name, v) )
continue
if k not in LIT_METRIC_TO_LNT or LIT_METRIC_TO_LNT[k] in ignore:
continue
test_samples.append(
lnt.testing.TestSamples(name + '.' + LIT_METRIC_TO_LNT[k],
[v],
test_info,
LIT_METRIC_CONV_FN[k]))
if self._test_failed_to_compile(raw_name, path):
test_samples.append(
lnt.testing.TestSamples(name + '.compile.status',
[lnt.testing.FAIL],
test_info))
elif not is_pass:
test_samples.append(
lnt.testing.TestSamples(name + '.exec.status',
[self._get_lnt_code(test_data['code'])],
test_info))
# Now import the profiles in parallel.
if profiles_to_import:
note('Importing %d profiles with %d threads...' %
(len(profiles_to_import), multiprocessing.cpu_count()))
TIMEOUT = 800
try:
pool = multiprocessing.Pool()
waiter = pool.map_async(_importProfile, profiles_to_import)
samples = waiter.get(TIMEOUT)
test_samples.extend([sample
for sample in samples
if sample is not None])
except multiprocessing.TimeoutError:
warning('Profiles had not completed importing after %s seconds.'
% TIMEOUT)
note('Aborting profile import and continuing')
if self.opts.single_result:
# If we got this far, the result we were looking for didn't exist.
raise RuntimeError("Result %s did not exist!" %
self.opts.single_result)
# FIXME: Add more machine info!
run_info = {
'tag': 'nts'
}
run_info.update(self._get_cc_info())
run_info['run_order'] = run_info['inferred_run_order']
if self.opts.run_order:
run_info['run_order'] = self.opts.run_order
machine_info = {
}
machine = lnt.testing.Machine(self.nick, machine_info)
run = lnt.testing.Run(self.start_time, timestamp(), info=run_info)
report = lnt.testing.Report(machine, run, test_samples)
return report
def log(self, message, ts=None):
if not ts:
ts = timestamp()
print >> sys.stderr, '%s: %s' % (ts, message)
def run_test(self, opts):
# Resolve the cc_under_test path.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
self._fatal('--cc does not point to a valid executable.')
# Attempt to infer the cxx compiler if not given.
if opts.cc and opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
logger.info("inferred C++ compiler under test as: %r" %
(opts.cxx, ))
if opts.cxx is None:
self._fatal('--cxx is required (and could not be inferred)')
# Force the CC and CXX variables to be absolute paths.
cc_abs = os.path.abspath(commands.which(opts.cc))
cxx_abs = os.path.abspath(commands.which(opts.cxx))
if not os.path.exists(cc_abs):
self._fatal("unable to determine absolute path for --cc: %r" %
(opts.cc, ))
if not os.path.exists(cxx_abs):
self._fatal("unable to determine absolute path for --cc: %r" %
(opts.cc, ))
opts.cc = cc_abs
opts.cxx = cxx_abs
# If no ld was set, set ld to opts.cc
if opts.ld is None:
opts.ld = opts.cc
# If no ldxx was set, set ldxx to opts.cxx
if opts.ldxx is None:
opts.ldxx = opts.cxx
# Set up the sandbox.
global g_output_dir
if not os.path.exists(opts.sandbox_path):
print >> sys.stderr, "%s: creating sandbox: %r" % (
timestamp(), opts.sandbox_path)
os.mkdir(opts.sandbox_path)
if opts.timestamp_build:
fmt_timestamp = timestamp().replace(' ', '_').replace(':', '-')
report_name = "test-%s" % (fmt_timestamp)
else:
report_name = "build"
g_output_dir = os.path.join(os.path.abspath(opts.sandbox_path),
report_name)
try:
os.mkdir(g_output_dir)
except OSError as e:
if e.errno == errno.EEXIST:
self._fatal("sandbox output directory %r already exists!" %
(g_output_dir, ))
else:
raise
# Setup log file
global g_log
def setup_log(output_dir):
def stderr_log_handler():
h = logging.StreamHandler()
f = logging.Formatter(
"%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
def file_log_handler(path):
h = logging.FileHandler(path, mode='w')
f = logging.Formatter(
"%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
log = logging.Logger('compile_test')
log.setLevel(logging.INFO)
log.addHandler(
file_log_handler(os.path.join(output_dir, 'test.log')))
log.addHandler(stderr_log_handler())
return log
g_log = setup_log(g_output_dir)
# Collect machine and run information.
machine_info, run_info = machineinfo.get_machine_information(
opts.use_machdep_info)
# FIXME: Include information on test source versions.
#
# FIXME: Get more machine information? Cocoa.h hash, for example.
for name, cmd in (('sys_cc_version', ('/usr/bin/gcc', '-v')),
('sys_as_version', ('/usr/bin/as', '-v',
'/dev/null')),
('sys_ld_version', ('/usr/bin/ld', '-v')),
('sys_xcodebuild', ('xcodebuild', '-version'))):
run_info[name] = commands.capture(cmd, include_stderr=True).strip()
# Set command line machine and run information.
for info, params in ((machine_info, opts.machine_parameters),
(run_info, opts.run_parameters)):
for entry in params:
if '=' not in entry:
name, value = entry, ''
else:
name, value = entry.split('=', 1)
info[name] = value
# Set user variables.
variables = {}
variables['cc'] = opts.cc
variables['run_count'] = opts.run_count
# Get compiler info.
cc_info = lnt.testing.util.compilers.get_cc_info(variables['cc'])
variables.update(cc_info)
# Set the run order from the user, if given.
if opts.run_order is not None:
variables['run_order'] = opts.run_order
else:
# Otherwise, use the inferred run order.
variables['run_order'] = cc_info['inferred_run_order']
logger.info("inferred run order to be: %r" %
(variables['run_order'], ))
if opts.verbose:
format = pprint.pformat(variables)
msg = '\n\t'.join(['using variables:'] + format.splitlines())
logger.info(msg)
format = pprint.pformat(machine_info)
msg = '\n\t'.join(['using machine info:'] + format.splitlines())
logger.info(msg)
format = pprint.pformat(run_info)
msg = '\n\t'.join(['using run info:'] + format.splitlines())
logger.info(msg)
# Compute the set of flags to test.
if not opts.flags_to_test:
flags_to_test = DEFAULT_FLAGS_TO_TEST
else:
flags_to_test = [
string.split(' ') for string in opts.flags_to_test
]
# Compute the set of job counts to use in full build tests.
if not opts.jobs_to_test:
jobs_to_test = [1, 2, 4, 8]
else:
jobs_to_test = opts.jobs_to_test
# Compute the build configurations to test.
if not opts.configs_to_test:
configs_to_test = ['Debug', 'Release']
else:
configs_to_test = opts.configs_to_test
# Compute the list of all tests.
all_tests = list(
get_tests(opts.test_suite_externals, opts.test_subdir,
flags_to_test, jobs_to_test, configs_to_test))
# Show the tests, if requested.
if opts.show_tests:
print >> sys.stderr, 'Available Tests'
for name in sorted(set(name for name, _ in all_tests)):
print >> sys.stderr, ' %s' % (name, )
print
raise SystemExit
# Find the tests to run.
if not opts.tests and not opts.test_filters:
tests_to_run = list(all_tests)
else:
all_test_names = set(test[0] for test in all_tests)
# Validate the test names.
requested_tests = set(opts.tests)
missing_tests = requested_tests - all_test_names
if missing_tests:
self._fatal(("invalid test names %s, use --show-tests to "
"see available tests") %
(", ".join(map(repr, missing_tests)), ))
# Validate the test filters.
test_filters = [
re.compile(pattern) for pattern in opts.test_filters
]
# Form the list of tests.
tests_to_run = [
test for test in all_tests if
(test[0] in requested_tests or
[True for filter in test_filters if filter.search(test[0])])
]
if not tests_to_run:
self._fatal("no tests requested "
"(invalid --test or --test-filter options)!")
# Ensure output directory is available.
if not os.path.exists(g_output_dir):
os.mkdir(g_output_dir)
# Execute the run.
run_info.update(variables)
run_info['tag'] = tag = 'compile'
testsamples = []
start_time = datetime.utcnow()
g_log.info('run started')
g_log.info('using CC: %r' % opts.cc)
g_log.info('using CXX: %r' % opts.cxx)
no_errors = True
for basename, test_fn in tests_to_run:
for success, name, samples in test_fn(basename, run_info,
variables):
g_log.info('collected samples: %r' % name)
num_samples = len(samples)
if num_samples:
samples_median = '%.4f' % (stats.median(samples), )
samples_mad = '%.4f' % (
stats.median_absolute_deviation(samples), )
else:
samples_median = samples_mad = 'N/A'
g_log.info('N=%d, median=%s, MAD=%s' %
(num_samples, samples_median, samples_mad))
test_name = '%s.%s' % (tag, name)
if not success:
testsamples.append(
lnt.testing.TestSamples(test_name + '.status',
[lnt.testing.FAIL]))
no_errors = False
if samples:
testsamples.append(
lnt.testing.TestSamples(test_name, samples))
run_info['no_errors'] = no_errors
end_time = datetime.utcnow()
g_log.info('run complete')
# Package up the report.
machine = lnt.testing.Machine(opts.machine_name, machine_info)
run = lnt.testing.Run(start_time, end_time, info=run_info)
# Write out the report.
lnt_report_path = os.path.join(g_output_dir, 'report.json')
report = lnt.testing.Report(machine, run, testsamples)
# Save report to disk for submission.
self.print_report(report, lnt_report_path)
# Then, also print to screen if requested.
if opts.output is not None:
self.print_report(report, opts.output)
server_report = self.submit(lnt_report_path, opts, ts_name='compile')
return server_report
def _parse_lit_output(self, path, data, cmake_vars, only_test=False):
LIT_METRIC_TO_LNT = {
'compile_time': 'compile',
'exec_time': 'exec',
'score': 'score',
'hash': 'hash',
'link_time': 'compile',
'size.__text': 'code_size',
'mem_bytes': 'mem',
'link_mem_bytes': 'mem'
}
LIT_METRIC_CONV_FN = {
'compile_time': float,
'exec_time': float,
'score': float,
'hash': str,
'link_time': float,
'size.__text': float,
'mem_bytes': float,
'link_mem_bytes': float
}
# We don't use the test info, currently.
test_info = {}
test_samples = []
# FIXME: Populate with keys not to upload
ignore = self.opts.exclude_stat_from_submission
if only_test:
ignore.append('compile')
profiles_to_import = []
no_errors = True
for test_data in data['tests']:
code = test_data['code']
raw_name = test_data['name']
split_name = raw_name.split(' :: ', 1)
if len(split_name) > 1:
name = split_name[1]
else:
name = split_name[0]
if name.endswith('.test'):
name = name[:-5]
name = 'nts.' + name
# If --single-result is given, exit based on
# --single-result-predicate
is_pass = self._is_pass_code(code)
if self.opts.single_result and \
raw_name == self.opts.single_result + '.test':
env = {'status': is_pass}
if 'metrics' in test_data:
for k, v in test_data['metrics'].items():
env[k] = v
if k in LIT_METRIC_TO_LNT:
env[LIT_METRIC_TO_LNT[k]] = v
status = eval(self.opts.single_result_predicate, {}, env)
sys.exit(0 if status else 1)
if 'metrics' in test_data:
for k, v in sorted(test_data['metrics'].items()):
if k == 'profile':
profiles_to_import.append((name, v))
continue
if k not in LIT_METRIC_TO_LNT or \
LIT_METRIC_TO_LNT[k] in ignore:
continue
server_name = name + '.' + LIT_METRIC_TO_LNT[k]
if k == 'link_time' or k == 'link_mem_bytes':
# Move link time into a second benchmark's
# compile-time.
server_name = name + '-link.' + LIT_METRIC_TO_LNT[k]
test_samples.append(
lnt.testing.TestSamples(server_name,
[v],
test_info,
LIT_METRIC_CONV_FN[k]))
if code == 'NOEXE':
test_samples.append(
lnt.testing.TestSamples(name + '.compile.status',
[lnt.testing.FAIL],
test_info))
no_errors = False
elif not is_pass:
lnt_code = self._get_lnt_code(test_data['code'])
test_samples.append(
lnt.testing.TestSamples(name + '.exec.status',
[lnt_code], test_info))
no_errors = False
# Now import the profiles in parallel.
if profiles_to_import:
logger.info('Importing %d profiles with %d threads...' %
(len(profiles_to_import), multiprocessing.cpu_count()))
TIMEOUT = 800
try:
pool = multiprocessing.Pool()
waiter = pool.map_async(_importProfile, profiles_to_import)
samples = waiter.get(TIMEOUT)
test_samples.extend([sample
for sample in samples
if sample is not None])
except multiprocessing.TimeoutError:
logger.warning('Profiles had not completed importing after ' +
'%s seconds.' % TIMEOUT)
logger.info('Aborting profile import and continuing')
if self.opts.single_result:
# If we got this far, the result we were looking for didn't exist.
raise RuntimeError("Result %s did not exist!" %
self.opts.single_result)
# FIXME: Add more machine info!
run_info = {
'tag': 'nts',
'no_errors': no_errors,
}
run_info.update(self._get_cc_info(cmake_vars))
run_info['run_order'] = run_info['inferred_run_order']
if self.opts.run_order:
run_info['run_order'] = self.opts.run_order
machine_info = {
}
machine = lnt.testing.Machine(self.opts.label, machine_info)
run = lnt.testing.Run(self.start_time, timestamp(), info=run_info)
report = lnt.testing.Report(machine, run, test_samples)
return report
def run_test(self, opts):
if opts.cc is not None:
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
self._fatal('--cc does not point to a valid executable.')
# If there was no --cxx given, attempt to infer it from the --cc.
if opts.cxx is None:
opts.cxx = \
lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
logger.info("Inferred C++ compiler under test as: %r" %
(opts.cxx, ))
else:
self._fatal("unable to infer --cxx - set it manually.")
else:
opts.cxx = resolve_command_path(opts.cxx)
if not os.path.exists(opts.cxx):
self._fatal("invalid --cxx argument %r, does not exist" %
(opts.cxx))
if opts.test_suite_root is None:
self._fatal('--test-suite is required')
if not os.path.exists(opts.test_suite_root):
self._fatal("invalid --test-suite argument, does not exist: %r" %
(opts.test_suite_root))
opts.test_suite_root = os.path.abspath(opts.test_suite_root)
if opts.test_suite_externals:
if not os.path.exists(opts.test_suite_externals):
self._fatal(
"invalid --test-externals argument, does not exist: %r" %
(opts.test_suite_externals, ))
opts.test_suite_externals = os.path.abspath(
opts.test_suite_externals)
opts.cmake = resolve_command_path(opts.cmake)
if not isexecfile(opts.cmake):
self._fatal("CMake tool not found (looked for %s)" % opts.cmake)
opts.make = resolve_command_path(opts.make)
if not isexecfile(opts.make):
self._fatal("Make tool not found (looked for %s)" % opts.make)
opts.lit = resolve_command_path(opts.lit)
if not isexecfile(opts.lit):
self._fatal("LIT tool not found (looked for %s)" % opts.lit)
if opts.run_under:
split = shlex.split(opts.run_under)
split[0] = resolve_command_path(split[0])
if not isexecfile(split[0]):
self._fatal("Run under wrapper not found (looked for %s)" %
opts.run_under)
if opts.single_result:
# --single-result implies --only-test
opts.only_test = opts.single_result
if opts.only_test:
# --only-test can either point to a particular test or a directory.
# Therefore, test_suite_root + opts.only_test or
# test_suite_root + dirname(opts.only_test) must be a directory.
path = os.path.join(opts.test_suite_root, opts.only_test)
parent_path = os.path.dirname(path)
if os.path.isdir(path):
opts.only_test = (opts.only_test, None)
elif os.path.isdir(parent_path):
opts.only_test = (os.path.dirname(opts.only_test),
os.path.basename(opts.only_test))
else:
self._fatal("--only-test argument not understood (must be a " +
" test or directory name)")
if opts.single_result and not opts.only_test[1]:
self._fatal("--single-result must be given a single test name, "
"not a directory name")
opts.cppflags = ' '.join(opts.cppflags)
opts.cflags = ' '.join(opts.cflags)
opts.cxxflags = ' '.join(opts.cxxflags)
if opts.diagnose:
if not opts.only_test:
self._fatal("--diagnose requires --only-test")
self.start_time = timestamp()
# Work out where to put our build stuff
if opts.timestamp_build:
ts = self.start_time.replace(' ', '_').replace(':', '-')
build_dir_name = "test-%s" % ts
else:
build_dir_name = "build"
basedir = os.path.join(opts.sandbox_path, build_dir_name)
self._base_path = basedir
cmakecache = os.path.join(self._base_path, 'CMakeCache.txt')
self.configured = not opts.run_configure and \
os.path.exists(cmakecache)
# If we are doing diagnostics, skip the usual run and do them now.
if opts.diagnose:
return self.diagnose()
# configure, so we can extract toolchain information from the cmake
# output.
self._configure_if_needed()
# Verify that we can actually find a compiler before continuing
cmake_vars = self._extract_cmake_vars_from_cache()
if "CMAKE_C_COMPILER" not in cmake_vars or \
not os.path.exists(cmake_vars["CMAKE_C_COMPILER"]):
self._fatal(
"Couldn't find C compiler (%s). Maybe you should specify --cc?"
% cmake_vars.get("CMAKE_C_COMPILER"))
# We don't support compiling without testing as we can't get compile-
# time numbers from LIT without running the tests.
if opts.compile_multisample > opts.exec_multisample:
logger.info("Increasing number of execution samples to %d" %
opts.compile_multisample)
opts.exec_multisample = opts.compile_multisample
if opts.auto_name:
# Construct the nickname from a few key parameters.
cc_info = self._get_cc_info(cmake_vars)
cc_nick = '%s_%s' % (cc_info['cc_name'], cc_info['cc_build'])
opts.label += "__%s__%s" %\
(cc_nick, cc_info['cc_target'].split('-')[0])
logger.info('Using nickname: %r' % opts.label)
# When we can't detect the clang version we use 0 instead. That
# is a horrible failure mode because all of our data ends up going
# to order 0. The user needs to give an order if we can't detect!
if opts.run_order is None:
cc_info = self._get_cc_info(cmake_vars)
if cc_info['inferred_run_order'] == 0:
fatal("Cannot detect compiler version. Specify --run-order"
" to manually define it.")
# Now do the actual run.
reports = []
json_reports = []
for i in range(max(opts.exec_multisample, opts.compile_multisample)):
c = i < opts.compile_multisample
e = i < opts.exec_multisample
# only gather perf profiles on a single run.
p = i == 0 and opts.use_perf in ('profile', 'all')
run_report, json_data = self.run(cmake_vars,
compile=c,
test=e,
profile=p)
reports.append(run_report)
json_reports.append(json_data)
report = self._create_merged_report(reports)
# Write the report out so it can be read by the submission tool.
report_path = os.path.join(self._base_path, 'report.json')
with open(report_path, 'w') as fd:
fd.write(report.render())
if opts.output:
with open(opts.output, 'w') as fd:
fd.write(report.render())
xml_report_path = os.path.join(self._base_path,
'test-results.xunit.xml')
str_template = _lit_json_to_xunit_xml(json_reports)
with open(xml_report_path, 'w') as fd:
fd.write(str_template)
csv_report_path = os.path.join(self._base_path, 'test-results.csv')
str_template = _lit_json_to_csv(json_reports)
with open(csv_report_path, 'w') as fd:
fd.write(str_template)
return self.submit(report_path, opts, 'nts')
def _parse_lit_output(self, path, data, only_test=False):
LIT_METRIC_TO_LNT = {
'compile_time': 'compile',
'exec_time': 'exec',
'score': 'score',
'hash': 'hash'
}
LIT_METRIC_CONV_FN = {
'compile_time': float,
'exec_time': float,
'score': float,
'hash': str
}
# We don't use the test info, currently.
test_info = {}
test_samples = []
# FIXME: Populate with keys not to upload
ignore = self.opts.exclude_stat_from_submission
if only_test:
ignore.append('compile')
profiles_to_import = []
for test_data in data['tests']:
raw_name = test_data['name'].split(' :: ', 1)[1]
name = 'nts.' + raw_name.rsplit('.test', 1)[0]
is_pass = self._is_pass_code(test_data['code'])
# If --single-result is given, exit based on --single-result-predicate
if self.opts.single_result and \
raw_name == self.opts.single_result+'.test':
env = {'status': is_pass}
if 'metrics' in test_data:
for k, v in test_data['metrics'].items():
env[k] = v
if k in LIT_METRIC_TO_LNT:
env[LIT_METRIC_TO_LNT[k]] = v
status = eval(self.opts.single_result_predicate, {}, env)
sys.exit(0 if status else 1)
if 'metrics' in test_data:
for k, v in test_data['metrics'].items():
if k == 'profile':
profiles_to_import.append((name, v))
continue
if k not in LIT_METRIC_TO_LNT or LIT_METRIC_TO_LNT[
k] in ignore:
continue
test_samples.append(
lnt.testing.TestSamples(
name + '.' + LIT_METRIC_TO_LNT[k], [v], test_info,
LIT_METRIC_CONV_FN[k]))
if self._test_failed_to_compile(raw_name, path):
test_samples.append(
lnt.testing.TestSamples(name + '.compile.status',
[lnt.testing.FAIL], test_info))
elif not is_pass:
test_samples.append(
lnt.testing.TestSamples(
name + '.exec.status',
[self._get_lnt_code(test_data['code'])], test_info))
# Now import the profiles in parallel.
if profiles_to_import:
note('Importing %d profiles with %d threads...' %
(len(profiles_to_import), multiprocessing.cpu_count()))
TIMEOUT = 800
try:
pool = multiprocessing.Pool()
waiter = pool.map_async(_importProfile, profiles_to_import)
samples = waiter.get(TIMEOUT)
test_samples.extend(
[sample for sample in samples if sample is not None])
except multiprocessing.TimeoutError:
warning(
'Profiles had not completed importing after %s seconds.' %
TIMEOUT)
note('Aborting profile import and continuing')
if self.opts.single_result:
# If we got this far, the result we were looking for didn't exist.
raise RuntimeError("Result %s did not exist!" %
self.opts.single_result)
# FIXME: Add more machine info!
run_info = {'tag': 'nts'}
run_info.update(self._get_cc_info())
run_info['run_order'] = run_info['inferred_run_order']
if self.opts.run_order:
run_info['run_order'] = self.opts.run_order
machine_info = {}
machine = lnt.testing.Machine(self.nick, machine_info)
run = lnt.testing.Run(self.start_time, timestamp(), info=run_info)
report = lnt.testing.Report(machine, run, test_samples)
return report
def run_test(self, name, args):
global opts
parser = OptionParser(
("%(name)s [options] [<output file>]\n" +
usage_info) % locals())
parser.add_option("-s", "--sandbox", dest="sandbox_path",
help="Parent directory to build and run tests in",
type=str, default=None, metavar="PATH")
group = OptionGroup(parser, "Test Options")
group.add_option("", "--no-timestamp", dest="timestamp_build",
help="Don't timestamp build directory (for testing)",
action="store_false", default=True)
group.add_option("", "--cc", dest="cc", type='str',
help="Path to the compiler under test",
action="store", default=None)
group.add_option("", "--cxx", dest="cxx",
help="Path to the C++ compiler to test",
type=str, default=None)
group.add_option("", "--ld", dest="ld",
help="Path to the c linker to use. (Xcode Distinction)",
type=str, default=None)
group.add_option("", "--ldxx", dest="ldxx",
help="Path to the cxx linker to use. (Xcode Distinction)",
type=str, default=None)
group.add_option("", "--test-externals", dest="test_suite_externals",
help="Path to the LLVM test-suite externals",
type=str, default=None, metavar="PATH")
group.add_option("", "--machine-param", dest="machine_parameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the machine parameters",
type=str, action="append", default=[])
group.add_option("", "--run-param", dest="run_parameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the run parameters",
type=str, action="append", default=[])
group.add_option("", "--run-order", dest="run_order", metavar="STR",
help="String to use to identify and order this run",
action="store", type=str, default=None)
group.add_option("", "--test-subdir", dest="test_subdir",
help="Subdirectory of test external dir to look for tests in.",
type=str, default="lnt-compile-suite-src")
parser.add_option_group(group)
group = OptionGroup(parser, "Test Selection")
group.add_option("", "--no-memory-profiling", dest="memory_profiling",
help="Disable memory profiling",
action="store_false", default=True)
group.add_option("", "--multisample", dest="run_count", metavar="N",
help="Accumulate test data from multiple runs",
action="store", type=int, default=3)
group.add_option("", "--min-sample-time", dest="min_sample_time",
help="Ensure all tests run for at least N seconds",
metavar="N", action="store", type=float, default=.5)
group.add_option("", "--save-temps", dest="save_temps",
help="Save temporary build output files",
action="store_true", default=False)
group.add_option("", "--show-tests", dest="show_tests",
help="Only list the availables tests that will be run",
action="store_true", default=False)
group.add_option("", "--test", dest="tests", metavar="NAME",
help="Individual test to run",
action="append", default=[])
group.add_option("", "--test-filter", dest="test_filters",
help="Run tests matching the given pattern",
metavar="REGEXP", action="append", default=[])
group.add_option("", "--flags-to-test", dest="flags_to_test",
help="Add a set of flags to test (space separated)",
metavar="FLAGLIST", action="append", default=[])
group.add_option("", "--jobs-to-test", dest="jobs_to_test",
help="Add a job count to test (full builds)",
metavar="NUM", action="append", default=[], type=int)
group.add_option("", "--config-to-test", dest="configs_to_test",
help="Add build configuration to test (full builds)",
metavar="NAME", action="append", default=[],
choices=('Debug', 'Release'))
parser.add_option_group(group)
group = OptionGroup(parser, "Output Options")
group.add_option("", "--no-machdep-info", dest="use_machdep_info",
help=("Don't put machine (instance) dependent "
"variables in machine info"),
action="store_false", default=True)
group.add_option("", "--machine-name", dest="machine_name", type='str',
help="Machine name to use in submission [%default]",
action="store", default=platform.uname()[1])
group.add_option("", "--submit", dest="submit_url", metavar="URLORPATH",
help=("autosubmit the test result to the given server "
"(or local instance) [%default]"),
type=str, default=None)
group.add_option("", "--commit", dest="commit",
help=("whether the autosubmit result should be committed "
"[%default]"),
type=int, default=True)
group.add_option("", "--output", dest="output", metavar="PATH",
help="write raw report data to PATH (or stdout if '-')",
action="store", default=None)
group.add_option("-v", "--verbose", dest="verbose",
help="show verbose test results",
action="store_true", default=False)
parser.add_option_group(group)
opts, args = parser.parse_args(args)
if len(args) != 0:
parser.error("invalid number of arguments")
if opts.cc is None:
parser.error("You must specify a --cc argument.")
# Resolve the cc_under_test path.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
parser.error('--cc does not point to a valid executable.')
# Attempt to infer the cxx compiler if not given.
if opts.cc and opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
note("inferred C++ compiler under test as: %r" % (opts.cxx,))
# Validate options.
if opts.cc is None:
parser.error('--cc is required')
if opts.cxx is None:
parser.error('--cxx is required (and could not be inferred)')
if opts.sandbox_path is None:
parser.error('--sandbox is required')
if opts.test_suite_externals is None:
parser.error("--test-externals option is required")
# Force the CC and CXX variables to be absolute paths.
cc_abs = os.path.abspath(commands.which(opts.cc))
cxx_abs = os.path.abspath(commands.which(opts.cxx))
if not os.path.exists(cc_abs):
parser.error("unable to determine absolute path for --cc: %r" % (
opts.cc,))
if not os.path.exists(cxx_abs):
parser.error("unable to determine absolute path for --cc: %r" % (
opts.cc,))
opts.cc = cc_abs
opts.cxx = cxx_abs
# If no ld was set, set ld to opts.cc
if opts.ld is None:
opts.ld = opts.cc
# If no ldxx was set, set ldxx to opts.cxx
if opts.ldxx is None:
opts.ldxx = opts.cxx
# Set up the sandbox.
global g_output_dir
if not os.path.exists(opts.sandbox_path):
print >>sys.stderr, "%s: creating sandbox: %r" % (
timestamp(), opts.sandbox_path)
os.mkdir(opts.sandbox_path)
if opts.timestamp_build:
fmt_timestamp = timestamp().replace(' ', '_').replace(':', '-')
report_name = "test-%s" % (fmt_timestamp)
else:
report_name = "build"
g_output_dir = os.path.join(os.path.abspath(opts.sandbox_path),
report_name)
try:
os.mkdir(g_output_dir)
except OSError(e):
if e.errno == errno.EEXIST:
parser.error("sandbox output directory %r already exists!" % (
g_output_dir,))
else:
raise
# Setup log file
global g_log
def setup_log(output_dir):
def stderr_log_handler():
h = logging.StreamHandler()
f = logging.Formatter("%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
def file_log_handler(path):
h = logging.FileHandler(path, mode='w')
f = logging.Formatter("%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
l = logging.Logger('compile_test')
l.setLevel(logging.INFO)
l.addHandler(file_log_handler(os.path.join(output_dir, 'test.log')))
l.addHandler(stderr_log_handler())
return l
g_log = setup_log(g_output_dir)
# Collect machine and run information.
machine_info, run_info = machineinfo.get_machine_information(
opts.use_machdep_info)
# FIXME: Include information on test source versions.
#
# FIXME: Get more machine information? Cocoa.h hash, for example.
for name, cmd in (('sys_cc_version', ('/usr/bin/gcc', '-v')),
('sys_as_version', ('/usr/bin/as', '-v', '/dev/null')),
('sys_ld_version', ('/usr/bin/ld', '-v')),
('sys_xcodebuild', ('xcodebuild', '-version'))):
run_info[name] = commands.capture(cmd, include_stderr=True).strip()
# Set command line machine and run information.
for info, params in ((machine_info, opts.machine_parameters),
(run_info, opts.run_parameters)):
for entry in params:
if '=' not in entry:
name, value = entry, ''
else:
name, value = entry.split('=', 1)
info[name] = value
# Set user variables.
variables = {}
variables['cc'] = opts.cc
variables['run_count'] = opts.run_count
# Get compiler info.
cc_info = lnt.testing.util.compilers.get_cc_info(variables['cc'])
variables.update(cc_info)
# Set the run order from the user, if given.
if opts.run_order is not None:
variables['run_order'] = opts.run_order
else:
# Otherwise, use the inferred run order.
variables['run_order'] = cc_info['inferred_run_order']
note("inferred run order to be: %r" % (variables['run_order'],))
if opts.verbose:
format = pprint.pformat(variables)
msg = '\n\t'.join(['using variables:'] + format.splitlines())
note(msg)
format = pprint.pformat(machine_info)
msg = '\n\t'.join(['using machine info:'] + format.splitlines())
note(msg)
format = pprint.pformat(run_info)
msg = '\n\t'.join(['using run info:'] + format.splitlines())
note(msg)
# Compute the set of flags to test.
if not opts.flags_to_test:
flags_to_test = DEFAULT_FLAGS_TO_TEST
else:
flags_to_test = [string.split(' ')
for string in opts.flags_to_test]
# Compute the set of job counts to use in full build tests.
if not opts.jobs_to_test:
jobs_to_test = [1, 2, 4, 8]
else:
jobs_to_test = opts.jobs_to_test
# Compute the build configurations to test.
if not opts.configs_to_test:
configs_to_test = ['Debug', 'Release']
else:
configs_to_test = opts.configs_to_test
# Compute the list of all tests.
all_tests = list(get_tests(opts.test_suite_externals, opts.test_subdir,
flags_to_test, jobs_to_test,
configs_to_test))
# Show the tests, if requested.
if opts.show_tests:
print >>sys.stderr, 'Available Tests'
for name in sorted(set(name for name, _ in all_tests)):
print >>sys.stderr, ' %s' % (name,)
print
raise SystemExit
# Find the tests to run.
if not opts.tests and not opts.test_filters:
tests_to_run = list(all_tests)
else:
all_test_names = set(test[0] for test in all_tests)
# Validate the test names.
requested_tests = set(opts.tests)
missing_tests = requested_tests - all_test_names
if missing_tests:
parser.error(("invalid test names %s, use --show-tests to "
"see available tests") %
(", ".join(map(repr, missing_tests)), ))
# Validate the test filters.
test_filters = [re.compile(pattern)
for pattern in opts.test_filters]
# Form the list of tests.
tests_to_run = [test
for test in all_tests
if (test[0] in requested_tests or
[True
for filter in test_filters
if filter.search(test[0])])]
if not tests_to_run:
parser.error(
"no tests requested (invalid --test or --test-filter options)!")
# Ensure output directory is available.
if not os.path.exists(g_output_dir):
os.mkdir(g_output_dir)
# Execute the run.
run_info.update(variables)
run_info['tag'] = tag = 'compile'
testsamples = []
start_time = datetime.utcnow()
g_log.info('run started')
g_log.info('using CC: %r' % opts.cc)
g_log.info('using CXX: %r' % opts.cxx)
for basename, test_fn in tests_to_run:
for success, name, samples in test_fn(basename, run_info,
variables):
g_log.info('collected samples: %r' % name)
num_samples = len(samples)
if num_samples:
samples_median = '%.4f' % (stats.median(samples),)
samples_mad = '%.4f' % (
stats.median_absolute_deviation(samples),)
else:
samples_median = samples_mad = 'N/A'
g_log.info('N=%d, median=%s, MAD=%s' % (
num_samples, samples_median, samples_mad))
test_name = '%s.%s' % (tag, name)
if not success:
testsamples.append(lnt.testing.TestSamples(
test_name + '.status',
[lnt.testing.FAIL]))
if samples:
testsamples.append(lnt.testing.TestSamples(
test_name, samples))
end_time = datetime.utcnow()
g_log.info('run complete')
# Package up the report.
machine = lnt.testing.Machine(opts.machine_name, machine_info)
run = lnt.testing.Run(start_time, end_time, info=run_info)
# Write out the report.
lnt_report_path = os.path.join(g_output_dir, 'report.json')
report = lnt.testing.Report(machine, run, testsamples)
# Save report to disk for submission.
self.print_report(report, lnt_report_path)
# Then, also print to screen if requested.
if opts.output is not None:
self.print_report(report, opts.output)
server_report = self.submit(lnt_report_path, opts)
return server_report
def log(self, message, ts=timestamp()):
print >>sys.stderr, '%s: %s' % (ts, message)
def log(self, message, ts=timestamp()):
print >> sys.stderr, '%s: %s' % (ts, message)
def run_test(self, name, args):
# FIXME: Add more detailed usage information
parser = OptionParser("%s [options] test-suite" % name)
group = OptionGroup(parser, "Sandbox options")
group.add_option("-S",
"--sandbox",
dest="sandbox_path",
help="Parent directory to build and run tests in",
type=str,
default=None,
metavar="PATH")
group.add_option("",
"--no-timestamp",
dest="timestamp_build",
action="store_false",
default=True,
help="Don't timestamp build directory (for testing)")
group.add_option("",
"--no-configure",
dest="run_configure",
action="store_false",
default=True,
help="Don't run CMake if CMakeCache.txt is present"
" (only useful with --no-timestamp")
parser.add_option_group(group)
group = OptionGroup(parser, "Inputs")
group.add_option("",
"--test-suite",
dest="test_suite_root",
type=str,
metavar="PATH",
default=None,
help="Path to the LLVM test-suite sources")
group.add_option("",
"--test-externals",
dest="test_suite_externals",
type=str,
metavar="PATH",
help="Path to the LLVM test-suite externals")
parser.add_option_group(group)
group = OptionGroup(parser, "Test compiler")
group.add_option("",
"--cc",
dest="cc",
metavar="CC",
type=str,
default=None,
help="Path to the C compiler to test")
group.add_option("",
"--cxx",
dest="cxx",
metavar="CXX",
type=str,
default=None,
help="Path to the C++ compiler to test (inferred from"
" --cc where possible")
group.add_option("",
"--llvm-arch",
dest="llvm_arch",
type='choice',
default=None,
help="Override the CMake-inferred architecture",
choices=TEST_SUITE_KNOWN_ARCHITECTURES)
group.add_option("",
"--cross-compiling",
dest="cross_compiling",
action="store_true",
default=False,
help="Inform CMake that it should be cross-compiling")
group.add_option("",
"--cross-compiling-system-name",
type=str,
default=None,
dest="cross_compiling_system_name",
help="The parameter to pass to CMAKE_SYSTEM_NAME when"
" cross-compiling. By default this is 'Linux' "
"unless -arch is in the cflags, in which case "
"it is 'Darwin'")
group.add_option(
"",
"--cppflags",
type=str,
action="append",
dest="cppflags",
default=[],
help="Extra flags to pass the compiler in C or C++ mode. "
"Can be given multiple times")
group.add_option("",
"--cflags",
type=str,
action="append",
dest="cflags",
default=[],
help="Extra CFLAGS to pass to the compiler. Can be "
"given multiple times")
group.add_option("",
"--cxxflags",
type=str,
action="append",
dest="cxxflags",
default=[],
help="Extra CXXFLAGS to pass to the compiler. Can be "
"given multiple times")
parser.add_option_group(group)
group = OptionGroup(parser, "Test selection")
group.add_option("",
"--test-size",
type='choice',
dest="test_size",
choices=['small', 'regular', 'large'],
default='regular',
help="The size of test inputs to use")
group.add_option("",
"--benchmarking-only",
dest="benchmarking_only",
action="store_true",
default=False,
help="Benchmarking-only mode. Disable unit tests and "
"other flaky or short-running tests")
group.add_option("",
"--only-test",
dest="only_test",
metavar="PATH",
type=str,
default=None,
help="Only run tests under PATH")
parser.add_option_group(group)
group = OptionGroup(parser, "Test Execution")
group.add_option("-j",
"--threads",
dest="threads",
help="Number of testing threads",
type=int,
default=1,
metavar="N")
group.add_option("",
"--build-threads",
dest="build_threads",
help="Number of compilation threads",
type=int,
default=0,
metavar="N")
group.add_option("",
"--use-perf",
dest="use_perf",
help=("Use perf to obtain high accuracy timing"
"[%default]"),
type=str,
default=None)
group.add_option(
"",
"--exec-multisample",
dest="exec_multisample",
help="Accumulate execution test data from multiple runs",
type=int,
default=1,
metavar="N")
group.add_option(
"",
"--compile-multisample",
dest="compile_multisample",
help="Accumulate compile test data from multiple runs",
type=int,
default=1,
metavar="N")
parser.add_option_group(group)
group = OptionGroup(parser, "Output Options")
group.add_option("",
"--submit",
dest="submit_url",
metavar="URLORPATH",
help=("autosubmit the test result to the given server"
" (or local instance) [%default]"),
type=str,
default=None)
group.add_option(
"",
"--commit",
dest="commit",
help=("whether the autosubmit result should be committed "
"[%default]"),
type=int,
default=True)
group.add_option("-v",
"--verbose",
dest="verbose",
help="show verbose test results",
action="store_true",
default=False)
group.add_option("",
"--exclude-stat-from-submission",
dest="exclude_stat_from_submission",
help="Do not submit the stat of this type [%default]",
action='append',
choices=KNOWN_SAMPLE_KEYS,
type='choice',
default=['hash'])
parser.add_option_group(group)
group = OptionGroup(parser, "Test tools")
group.add_option("",
"--use-cmake",
dest="cmake",
metavar="PATH",
type=str,
default="cmake",
help="Path to CMake [cmake]")
group.add_option("",
"--use-make",
dest="make",
metavar="PATH",
type=str,
default="make",
help="Path to Make [make]")
group.add_option("",
"--use-lit",
dest="lit",
metavar="PATH",
type=str,
default="llvm-lit",
help="Path to the LIT test runner [llvm-lit]")
(opts, args) = parser.parse_args(args)
self.opts = opts
if args:
parser.error("Expected no positional arguments (got: %r)" %
(args, ))
for a in [
'cross_compiling', 'cross_compiling_system_name', 'llvm_arch',
'benchmarking_only', 'use_perf'
]:
if getattr(opts, a):
parser.error('option "%s" is not yet implemented!' % a)
if self.opts.sandbox_path is None:
parser.error('--sandbox is required')
# Option validation.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
parser.error('--cc does not point to a valid executable.')
# If there was no --cxx given, attempt to infer it from the --cc.
if opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
note("inferred C++ compiler under test as: %r" % (opts.cxx, ))
else:
parser.error("unable to infer --cxx - set it manually.")
if not os.path.exists(opts.cxx):
parser.error("invalid --cxx argument %r, does not exist" %
(opts.cxx))
if opts.test_suite_root is None:
parser.error('--test-suite is required')
if not os.path.exists(opts.test_suite_root):
parser.error("invalid --test-suite argument, does not exist: %r" %
(opts.test_suite_root))
if opts.test_suite_externals:
if not os.path.exists(opts.test_suite_externals):
parser.error(
"invalid --test-externals argument, does not exist: %r" %
(opts.test_suite_externals, ))
opts.cmake = resolve_command_path(opts.cmake)
if not isexecfile(opts.cmake):
parser.error("CMake tool not found (looked for %s)" % opts.cmake)
opts.make = resolve_command_path(opts.make)
if not isexecfile(opts.make):
parser.error("Make tool not found (looked for %s)" % opts.make)
opts.lit = resolve_command_path(opts.lit)
if not isexecfile(opts.lit):
parser.error("LIT tool not found (looked for %s)" % opts.lit)
opts.cppflags = ' '.join(opts.cppflags)
opts.cflags = ' '.join(opts.cflags)
opts.cxxflags = ' '.join(opts.cxxflags)
self.start_time = timestamp()
# Work out where to put our build stuff
if self.opts.timestamp_build:
ts = self.start_time.replace(' ', '_').replace(':', '-')
build_dir_name = "test-%s" % ts
else:
build_dir_name = "build"
basedir = os.path.join(self.opts.sandbox_path, build_dir_name)
self._base_path = basedir
# We don't support compiling without testing as we can't get compile-
# time numbers from LIT without running the tests.
if opts.compile_multisample > opts.exec_multisample:
note("Increasing number of execution samples to %d" %
opts.compile_multisample)
opts.exec_multisample = opts.compile_multisample
# Now do the actual run.
reports = []
for i in range(max(opts.exec_multisample, opts.compile_multisample)):
c = i < opts.compile_multisample
e = i < opts.exec_multisample
reports.append(self.run("FIXME: nick", compile=c, test=e))
report = self._create_merged_report(reports)
# Write the report out so it can be read by the submission tool.
report_path = os.path.join(self._base_path, 'report.json')
with open(report_path, 'w') as fd:
fd.write(report.render())
return self.submit(report_path, self.opts, commit=True)
def log(self, message, ts=None):
if not ts:
ts = timestamp()
print >>sys.stderr, '%s: %s' % (ts, message)
def _parse_lit_output(self, path, data, cmake_vars, only_test=False):
LIT_METRIC_TO_LNT = {
'compile_time': 'compile',
'exec_time': 'exec',
'score': 'score',
'hash': 'hash',
'link_time': 'compile',
'size.__text': 'code_size',
'mem_bytes': 'mem',
'link_mem_bytes': 'mem'
}
LIT_METRIC_CONV_FN = {
'compile_time': float,
'exec_time': float,
'score': float,
'hash': str,
'link_time': float,
'size.__text': float,
'mem_bytes': float,
'link_mem_bytes': float
}
# We don't use the test info, currently.
test_info = {}
test_samples = []
# FIXME: Populate with keys not to upload
ignore = self.opts.exclude_stat_from_submission
if only_test:
ignore.append('compile')
profiles_to_import = []
no_errors = True
for test_data in data['tests']:
code = test_data['code']
raw_name = test_data['name']
split_name = raw_name.split(' :: ', 1)
if len(split_name) > 1:
name = split_name[1]
else:
name = split_name[0]
if name.endswith('.test'):
name = name[:-5]
name = 'nts.' + name
# If --single-result is given, exit based on
# --single-result-predicate
is_pass = self._is_pass_code(code)
if self.opts.single_result and \
raw_name == self.opts.single_result + '.test':
env = {'status': is_pass}
if 'metrics' in test_data:
for k, v in test_data['metrics'].items():
env[k] = v
if k in LIT_METRIC_TO_LNT:
env[LIT_METRIC_TO_LNT[k]] = v
status = eval(self.opts.single_result_predicate, {}, env)
sys.exit(0 if status else 1)
if 'metrics' in test_data:
for k, v in sorted(test_data['metrics'].items()):
if k == 'profile':
profiles_to_import.append((name, v))
continue
if k not in LIT_METRIC_TO_LNT or \
LIT_METRIC_TO_LNT[k] in ignore:
continue
server_name = name + '.' + LIT_METRIC_TO_LNT[k]
if k == 'link_time' or k == 'link_mem_bytes':
# Move link time into a second benchmark's
# compile-time.
server_name = name + '-link.' + LIT_METRIC_TO_LNT[k]
test_samples.append(
lnt.testing.TestSamples(server_name, [v], test_info,
LIT_METRIC_CONV_FN[k]))
if code == 'NOEXE':
test_samples.append(
lnt.testing.TestSamples(name + '.compile.status',
[lnt.testing.FAIL], test_info))
no_errors = False
elif not is_pass:
lnt_code = self._get_lnt_code(test_data['code'])
test_samples.append(
lnt.testing.TestSamples(name + '.exec.status', [lnt_code],
test_info))
no_errors = False
# Now import the profiles in parallel.
if profiles_to_import:
logger.info('Importing %d profiles with %d threads...' %
(len(profiles_to_import), multiprocessing.cpu_count()))
TIMEOUT = 800
try:
pool = multiprocessing.Pool()
waiter = pool.map_async(_importProfile, profiles_to_import)
samples = waiter.get(TIMEOUT)
test_samples.extend(
[sample for sample in samples if sample is not None])
except multiprocessing.TimeoutError:
logger.warning('Profiles had not completed importing after ' +
'%s seconds.' % TIMEOUT)
logger.info('Aborting profile import and continuing')
if self.opts.single_result:
# If we got this far, the result we were looking for didn't exist.
raise RuntimeError("Result %s did not exist!" %
self.opts.single_result)
# FIXME: Add more machine info!
run_info = {
'tag': 'nts',
'no_errors': no_errors,
}
run_info.update(self._get_cc_info(cmake_vars))
run_info['run_order'] = run_info['inferred_run_order']
if self.opts.run_order:
run_info['run_order'] = self.opts.run_order
machine_info = {}
machine = lnt.testing.Machine(self.opts.label, machine_info)
run = lnt.testing.Run(self.start_time, timestamp(), info=run_info)
report = lnt.testing.Report(machine, run, test_samples)
return report
def run_test(self, name, args):
global opts
parser = OptionParser(
("%(name)s [options] [<output file>]\n" +
usage_info) % locals())
parser.add_option("-s", "--sandbox", dest="sandbox_path",
help="Parent directory to build and run tests in",
type=str, default=None, metavar="PATH")
group = OptionGroup(parser, "Test Options")
group.add_option("", "--no-timestamp", dest="timestamp_build",
help="Don't timestamp build directory (for testing)",
action="store_false", default=True)
group.add_option("", "--cc", dest="cc", type='str',
help="Path to the compiler under test",
action="store", default=None)
group.add_option("", "--cxx", dest="cxx",
help="Path to the C++ compiler to test",
type=str, default=None)
group.add_option("", "--ld", dest="ld",
help="Path to the c linker to use. (Xcode Distinction)",
type=str, default=None)
group.add_option("", "--ldxx", dest="ldxx",
help="Path to the cxx linker to use. (Xcode Distinction)",
type=str, default=None)
group.add_option("", "--test-externals", dest="test_suite_externals",
help="Path to the LLVM test-suite externals",
type=str, default=None, metavar="PATH")
group.add_option("", "--machine-param", dest="machine_parameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the machine parameters",
type=str, action="append", default=[])
group.add_option("", "--run-param", dest="run_parameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the run parameters",
type=str, action="append", default=[])
group.add_option("", "--run-order", dest="run_order", metavar="STR",
help="String to use to identify and order this run",
action="store", type=str, default=None)
group.add_option("", "--test-subdir", dest="test_subdir",
help="Subdirectory of test external dir to look for tests in.",
type=str, default="lnt-compile-suite-src")
parser.add_option_group(group)
group = OptionGroup(parser, "Test Selection")
group.add_option("", "--no-memory-profiling", dest="memory_profiling",
help="Disable memory profiling",
action="store_false", default=True)
group.add_option("", "--multisample", dest="run_count", metavar="N",
help="Accumulate test data from multiple runs",
action="store", type=int, default=3)
group.add_option("", "--min-sample-time", dest="min_sample_time",
help="Ensure all tests run for at least N seconds",
metavar="N", action="store", type=float, default=.5)
group.add_option("", "--save-temps", dest="save_temps",
help="Save temporary build output files",
action="store_true", default=False)
group.add_option("", "--show-tests", dest="show_tests",
help="Only list the availables tests that will be run",
action="store_true", default=False)
group.add_option("", "--test", dest="tests", metavar="NAME",
help="Individual test to run",
action="append", default=[])
group.add_option("", "--test-filter", dest="test_filters",
help="Run tests matching the given pattern",
metavar="REGEXP", action="append", default=[])
group.add_option("", "--flags-to-test", dest="flags_to_test",
help="Add a set of flags to test (space separated)",
metavar="FLAGLIST", action="append", default=[])
group.add_option("", "--jobs-to-test", dest="jobs_to_test",
help="Add a job count to test (full builds)",
metavar="NUM", action="append", default=[], type=int)
group.add_option("", "--config-to-test", dest="configs_to_test",
help="Add build configuration to test (full builds)",
metavar="NAME", action="append", default=[],
choices=('Debug','Release'))
parser.add_option_group(group)
group = OptionGroup(parser, "Output Options")
group.add_option("", "--no-machdep-info", dest="use_machdep_info",
help=("Don't put machine (instance) dependent "
"variables in machine info"),
action="store_false", default=True)
group.add_option("", "--machine-name", dest="machine_name", type='str',
help="Machine name to use in submission [%default]",
action="store", default=platform.uname()[1])
group.add_option("", "--submit", dest="submit_url", metavar="URLORPATH",
help=("autosubmit the test result to the given server "
"(or local instance) [%default]"),
type=str, default=None)
group.add_option("", "--commit", dest="commit",
help=("whether the autosubmit result should be committed "
"[%default]"),
type=int, default=True)
group.add_option("", "--output", dest="output", metavar="PATH",
help="write raw report data to PATH (or stdout if '-')",
action="store", default=None)
group.add_option("-v", "--verbose", dest="verbose",
help="show verbose test results",
action="store_true", default=False)
parser.add_option_group(group)
opts,args = parser.parse_args(args)
if len(args) != 0:
parser.error("invalid number of arguments")
if opts.cc is None:
parser.error("You must specify a --cc argument.")
# Resolve the cc_under_test path.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
parser.error('--cc does not point to a valid executable.')
# Attempt to infer the cxx compiler if not given.
if opts.cc and opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
note("inferred C++ compiler under test as: %r" % (opts.cxx,))
# Validate options.
if opts.cc is None:
parser.error('--cc is required')
if opts.cxx is None:
parser.error('--cxx is required (and could not be inferred)')
if opts.sandbox_path is None:
parser.error('--sandbox is required')
if opts.test_suite_externals is None:
parser.error("--test-externals option is required")
# Force the CC and CXX variables to be absolute paths.
cc_abs = os.path.abspath(commands.which(opts.cc))
cxx_abs = os.path.abspath(commands.which(opts.cxx))
if not os.path.exists(cc_abs):
parser.error("unable to determine absolute path for --cc: %r" % (
opts.cc,))
if not os.path.exists(cxx_abs):
parser.error("unable to determine absolute path for --cc: %r" % (
opts.cc,))
opts.cc = cc_abs
opts.cxx = cxx_abs
# If no ld was set, set ld to opts.cc
if opts.ld is None:
opts.ld = opts.cc
# If no ldxx was set, set ldxx to opts.cxx
if opts.ldxx is None:
opts.ldxx = opts.cxx
# Set up the sandbox.
global g_output_dir
if not os.path.exists(opts.sandbox_path):
print >>sys.stderr, "%s: creating sandbox: %r" % (
timestamp(), opts.sandbox_path)
os.mkdir(opts.sandbox_path)
if opts.timestamp_build:
report_name = "test-%s" % (timestamp().replace(' ','_').replace(':','-'))
else:
report_name = "build"
g_output_dir = os.path.join(os.path.abspath(opts.sandbox_path),report_name)
try:
os.mkdir(g_output_dir)
except OSError,e:
if e.errno == errno.EEXIST:
parser.error("sandbox output directory %r already exists!" % (
g_output_dir,))
else:
raise
def run_test(self, opts):
# Resolve the cc_under_test path.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
self._fatal('--cc does not point to a valid executable.')
# Attempt to infer the cxx compiler if not given.
if opts.cc and opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
logger.info("inferred C++ compiler under test as: %r" %
(opts.cxx,))
if opts.cxx is None:
self._fatal('--cxx is required (and could not be inferred)')
# Force the CC and CXX variables to be absolute paths.
cc_abs = os.path.abspath(commands.which(opts.cc))
cxx_abs = os.path.abspath(commands.which(opts.cxx))
if not os.path.exists(cc_abs):
self._fatal("unable to determine absolute path for --cc: %r" % (
opts.cc,))
if not os.path.exists(cxx_abs):
self._fatal("unable to determine absolute path for --cc: %r" % (
opts.cc,))
opts.cc = cc_abs
opts.cxx = cxx_abs
# If no ld was set, set ld to opts.cc
if opts.ld is None:
opts.ld = opts.cc
# If no ldxx was set, set ldxx to opts.cxx
if opts.ldxx is None:
opts.ldxx = opts.cxx
# Set up the sandbox.
global g_output_dir
if not os.path.exists(opts.sandbox_path):
print >>sys.stderr, "%s: creating sandbox: %r" % (
timestamp(), opts.sandbox_path)
os.mkdir(opts.sandbox_path)
if opts.timestamp_build:
fmt_timestamp = timestamp().replace(' ', '_').replace(':', '-')
report_name = "test-%s" % (fmt_timestamp)
else:
report_name = "build"
g_output_dir = os.path.join(os.path.abspath(opts.sandbox_path),
report_name)
try:
os.mkdir(g_output_dir)
except OSError as e:
if e.errno == errno.EEXIST:
self._fatal("sandbox output directory %r already exists!" % (
g_output_dir,))
else:
raise
# Setup log file
global g_log
def setup_log(output_dir):
def stderr_log_handler():
h = logging.StreamHandler()
f = logging.Formatter(
"%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
def file_log_handler(path):
h = logging.FileHandler(path, mode='w')
f = logging.Formatter(
"%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
log = logging.Logger('compile_test')
log.setLevel(logging.INFO)
log.addHandler(file_log_handler(os.path.join(output_dir,
'test.log')))
log.addHandler(stderr_log_handler())
return log
g_log = setup_log(g_output_dir)
# Collect machine and run information.
machine_info, run_info = machineinfo.get_machine_information(
opts.use_machdep_info)
# FIXME: Include information on test source versions.
#
# FIXME: Get more machine information? Cocoa.h hash, for example.
for name, cmd in (('sys_cc_version', ('/usr/bin/gcc', '-v')),
('sys_as_version',
('/usr/bin/as', '-v', '/dev/null')),
('sys_ld_version', ('/usr/bin/ld', '-v')),
('sys_xcodebuild', ('xcodebuild', '-version'))):
run_info[name] = commands.capture(cmd, include_stderr=True).strip()
# Set command line machine and run information.
for info, params in ((machine_info, opts.machine_parameters),
(run_info, opts.run_parameters)):
for entry in params:
if '=' not in entry:
name, value = entry, ''
else:
name, value = entry.split('=', 1)
info[name] = value
# Set user variables.
variables = {}
variables['cc'] = opts.cc
variables['run_count'] = opts.run_count
# Get compiler info.
cc_info = lnt.testing.util.compilers.get_cc_info(variables['cc'])
variables.update(cc_info)
# Set the run order from the user, if given.
if opts.run_order is not None:
variables['run_order'] = opts.run_order
else:
# Otherwise, use the inferred run order.
variables['run_order'] = cc_info['inferred_run_order']
logger.info("inferred run order to be: %r" %
(variables['run_order'],))
if opts.verbose:
format = pprint.pformat(variables)
msg = '\n\t'.join(['using variables:'] + format.splitlines())
logger.info(msg)
format = pprint.pformat(machine_info)
msg = '\n\t'.join(['using machine info:'] + format.splitlines())
logger.info(msg)
format = pprint.pformat(run_info)
msg = '\n\t'.join(['using run info:'] + format.splitlines())
logger.info(msg)
# Compute the set of flags to test.
if not opts.flags_to_test:
flags_to_test = DEFAULT_FLAGS_TO_TEST
else:
flags_to_test = [string.split(' ')
for string in opts.flags_to_test]
# Compute the set of job counts to use in full build tests.
if not opts.jobs_to_test:
jobs_to_test = [1, 2, 4, 8]
else:
jobs_to_test = opts.jobs_to_test
# Compute the build configurations to test.
if not opts.configs_to_test:
configs_to_test = ['Debug', 'Release']
else:
configs_to_test = opts.configs_to_test
# Compute the list of all tests.
all_tests = list(get_tests(opts.test_suite_externals, opts.test_subdir,
flags_to_test, jobs_to_test,
configs_to_test))
# Show the tests, if requested.
if opts.show_tests:
print >>sys.stderr, 'Available Tests'
for name in sorted(set(name for name, _ in all_tests)):
print >>sys.stderr, ' %s' % (name,)
print
raise SystemExit
# Find the tests to run.
if not opts.tests and not opts.test_filters:
tests_to_run = list(all_tests)
else:
all_test_names = set(test[0] for test in all_tests)
# Validate the test names.
requested_tests = set(opts.tests)
missing_tests = requested_tests - all_test_names
if missing_tests:
self._fatal(("invalid test names %s, use --show-tests to "
"see available tests") %
(", ".join(map(repr, missing_tests)), ))
# Validate the test filters.
test_filters = [re.compile(pattern)
for pattern in opts.test_filters]
# Form the list of tests.
tests_to_run = [test
for test in all_tests
if (test[0] in requested_tests or
[True
for filter in test_filters
if filter.search(test[0])])]
if not tests_to_run:
self._fatal("no tests requested "
"(invalid --test or --test-filter options)!")
# Ensure output directory is available.
if not os.path.exists(g_output_dir):
os.mkdir(g_output_dir)
# Execute the run.
run_info.update(variables)
run_info['tag'] = tag = 'compile'
testsamples = []
start_time = datetime.utcnow()
g_log.info('run started')
g_log.info('using CC: %r' % opts.cc)
g_log.info('using CXX: %r' % opts.cxx)
no_errors = True
for basename, test_fn in tests_to_run:
for success, name, samples in test_fn(basename, run_info,
variables):
g_log.info('collected samples: %r' % name)
num_samples = len(samples)
if num_samples:
samples_median = '%.4f' % (stats.median(samples),)
samples_mad = '%.4f' % (
stats.median_absolute_deviation(samples),)
else:
samples_median = samples_mad = 'N/A'
g_log.info('N=%d, median=%s, MAD=%s' % (
num_samples, samples_median, samples_mad))
test_name = '%s.%s' % (tag, name)
if not success:
testsamples.append(lnt.testing.TestSamples(
test_name + '.status', [lnt.testing.FAIL]))
no_errors = False
if samples:
testsamples.append(lnt.testing.TestSamples(
test_name, samples))
run_info['no_errors'] = no_errors
end_time = datetime.utcnow()
g_log.info('run complete')
# Package up the report.
machine = lnt.testing.Machine(opts.machine_name, machine_info)
run = lnt.testing.Run(start_time, end_time, info=run_info)
# Write out the report.
lnt_report_path = os.path.join(g_output_dir, 'report.json')
report = lnt.testing.Report(machine, run, testsamples)
# Save report to disk for submission.
self.print_report(report, lnt_report_path)
# Then, also print to screen if requested.
if opts.output is not None:
self.print_report(report, opts.output)
server_report = self.submit(lnt_report_path, opts, ts_name='compile')
return server_report
def run_test(self, name, args):
# FIXME: Add more detailed usage information
parser = OptionParser("%s [options] test-suite" % name)
group = OptionGroup(parser, "Sandbox options")
group.add_option("-S",
"--sandbox",
dest="sandbox_path",
help="Parent directory to build and run tests in",
type=str,
default=None,
metavar="PATH")
group.add_option("",
"--no-timestamp",
dest="timestamp_build",
action="store_false",
default=True,
help="Don't timestamp build directory (for testing)")
group.add_option("",
"--no-configure",
dest="run_configure",
action="store_false",
default=True,
help="Don't run CMake if CMakeCache.txt is present"
" (only useful with --no-timestamp")
parser.add_option_group(group)
group = OptionGroup(parser, "Inputs")
group.add_option("",
"--test-suite",
dest="test_suite_root",
type=str,
metavar="PATH",
default=None,
help="Path to the LLVM test-suite sources")
group.add_option("",
"--test-externals",
dest="test_suite_externals",
type=str,
metavar="PATH",
help="Path to the LLVM test-suite externals")
group.add_option(
"",
"--cmake-define",
dest="cmake_defines",
action="append",
help=("Defines to pass to cmake. These do not require the "
"-D prefix and can be given multiple times. e.g.: "
"--cmake-define A=B => -DA=B"))
group.add_option(
"-C",
"--cmake-cache",
dest="cmake_cache",
help=("Use one of the test-suite's cmake configurations."
" Ex: Release, Debug"))
parser.add_option_group(group)
group = OptionGroup(parser, "Test compiler")
group.add_option("",
"--cc",
dest="cc",
metavar="CC",
type=str,
default=None,
help="Path to the C compiler to test")
group.add_option("",
"--cxx",
dest="cxx",
metavar="CXX",
type=str,
default=None,
help="Path to the C++ compiler to test (inferred from"
" --cc where possible")
group.add_option("",
"--llvm-arch",
dest="llvm_arch",
type='choice',
default=None,
help="Override the CMake-inferred architecture",
choices=TEST_SUITE_KNOWN_ARCHITECTURES)
group.add_option("",
"--cross-compiling",
dest="cross_compiling",
action="store_true",
default=False,
help="Inform CMake that it should be cross-compiling")
group.add_option("",
"--cross-compiling-system-name",
type=str,
default=None,
dest="cross_compiling_system_name",
help="The parameter to pass to CMAKE_SYSTEM_NAME when"
" cross-compiling. By default this is 'Linux' "
"unless -arch is in the cflags, in which case "
"it is 'Darwin'")
group.add_option(
"",
"--cppflags",
type=str,
action="append",
dest="cppflags",
default=[],
help="Extra flags to pass the compiler in C or C++ mode. "
"Can be given multiple times")
group.add_option("",
"--cflags",
type=str,
action="append",
dest="cflags",
default=[],
help="Extra CFLAGS to pass to the compiler. Can be "
"given multiple times")
group.add_option("",
"--cxxflags",
type=str,
action="append",
dest="cxxflags",
default=[],
help="Extra CXXFLAGS to pass to the compiler. Can be "
"given multiple times")
parser.add_option_group(group)
group = OptionGroup(parser, "Test selection")
group.add_option("",
"--test-size",
type='choice',
dest="test_size",
choices=['small', 'regular', 'large'],
default='regular',
help="The size of test inputs to use")
group.add_option("",
"--benchmarking-only",
dest="benchmarking_only",
action="store_true",
default=False,
help="Benchmarking-only mode. Disable unit tests and "
"other flaky or short-running tests")
group.add_option("",
"--only-test",
dest="only_test",
metavar="PATH",
type=str,
default=None,
help="Only run tests under PATH")
parser.add_option_group(group)
group = OptionGroup(parser, "Test Execution")
group.add_option("-j",
"--threads",
dest="threads",
help="Number of testing (and optionally build) "
"threads",
type=int,
default=1,
metavar="N")
group.add_option("",
"--build-threads",
dest="build_threads",
help="Number of compilation threads, defaults to "
"--threads",
type=int,
default=0,
metavar="N")
group.add_option(
"",
"--use-perf",
dest="use_perf",
help=("Use Linux perf for high accuracy timing, profile "
"information or both"),
type='choice',
choices=['none', 'time', 'profile', 'all'],
default='none')
group.add_option("",
"--run-under",
dest="run_under",
help="Wrapper to run tests under ['%default']",
type=str,
default="")
group.add_option(
"",
"--exec-multisample",
dest="exec_multisample",
help="Accumulate execution test data from multiple runs",
type=int,
default=1,
metavar="N")
group.add_option(
"",
"--compile-multisample",
dest="compile_multisample",
help="Accumulate compile test data from multiple runs",
type=int,
default=1,
metavar="N")
group.add_option(
"-d",
"--diagnose",
dest="diagnose",
help="Produce a diagnostic report for a particular "
"test, this will not run all the tests. Must be"
" used in conjunction with --only-test.",
action="store_true",
default=False,
)
parser.add_option_group(group)
group = OptionGroup(parser, "Output Options")
group.add_option("",
"--no-auto-name",
dest="auto_name",
help="Don't automatically derive submission name",
action="store_false",
default=True)
group.add_option("",
"--run-order",
dest="run_order",
metavar="STR",
help="String to use to identify and order this run",
action="store",
type=str,
default=None)
group.add_option("",
"--submit",
dest="submit_url",
metavar="URLORPATH",
help=("autosubmit the test result to the given server"
" (or local instance) [%default]"),
type=str,
default=None)
group.add_option(
"",
"--commit",
dest="commit",
help=("whether the autosubmit result should be committed "
"[%default]"),
type=int,
default=True)
group.add_option("-v",
"--verbose",
dest="verbose",
help="show verbose test results",
action="store_true",
default=False)
group.add_option("",
"--succinct-compile-output",
help="run Make without VERBOSE=1",
action="store_true",
dest="succinct")
group.add_option("",
"--exclude-stat-from-submission",
dest="exclude_stat_from_submission",
help="Do not submit the stat of this type [%default]",
action='append',
choices=KNOWN_SAMPLE_KEYS,
type='choice',
default=[])
group.add_option("",
"--single-result",
dest="single_result",
help=("only execute this single test and apply "
"--single-result-predicate to calculate the "
"exit status"))
group.add_option("",
"--single-result-predicate",
dest="single_result_predicate",
help=("the predicate to apply to calculate the exit "
"status (with --single-result)"),
default="status")
parser.add_option_group(group)
group = OptionGroup(parser, "Test tools")
group.add_option("",
"--use-cmake",
dest="cmake",
metavar="PATH",
type=str,
default="cmake",
help="Path to CMake [cmake]")
group.add_option("",
"--use-make",
dest="make",
metavar="PATH",
type=str,
default="make",
help="Path to Make [make]")
group.add_option("",
"--use-lit",
dest="lit",
metavar="PATH",
type=str,
default="llvm-lit",
help="Path to the LIT test runner [llvm-lit]")
(opts, args) = parser.parse_args(args)
self.opts = opts
if len(args) == 0:
self.nick = platform.uname()[1]
elif len(args) == 1:
self.nick = args[0]
else:
parser.error("Expected no positional arguments (got: %r)" %
(args, ))
for a in [
'cross_compiling', 'cross_compiling_system_name', 'llvm_arch'
]:
if getattr(opts, a):
parser.error('option "%s" is not yet implemented!' % a)
if self.opts.sandbox_path is None:
parser.error('--sandbox is required')
# Option validation.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
parser.error('--cc does not point to a valid executable.')
# If there was no --cxx given, attempt to infer it from the --cc.
if opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
note("Inferred C++ compiler under test as: %r" % (opts.cxx, ))
else:
parser.error("unable to infer --cxx - set it manually.")
else:
opts.cxx = resolve_command_path(opts.cxx)
if not os.path.exists(opts.cxx):
parser.error("invalid --cxx argument %r, does not exist" %
(opts.cxx))
if opts.test_suite_root is None:
parser.error('--test-suite is required')
if not os.path.exists(opts.test_suite_root):
parser.error("invalid --test-suite argument, does not exist: %r" %
(opts.test_suite_root))
if opts.test_suite_externals:
if not os.path.exists(opts.test_suite_externals):
parser.error(
"invalid --test-externals argument, does not exist: %r" %
(opts.test_suite_externals, ))
opts.cmake = resolve_command_path(opts.cmake)
if not isexecfile(opts.cmake):
parser.error("CMake tool not found (looked for %s)" % opts.cmake)
opts.make = resolve_command_path(opts.make)
if not isexecfile(opts.make):
parser.error("Make tool not found (looked for %s)" % opts.make)
opts.lit = resolve_command_path(opts.lit)
if not isexecfile(opts.lit):
parser.error("LIT tool not found (looked for %s)" % opts.lit)
if opts.run_under:
split = shlex.split(opts.run_under)
split[0] = resolve_command_path(split[0])
if not isexecfile(split[0]):
parser.error("Run under wrapper not found (looked for %s)" %
opts.run_under)
if opts.single_result:
# --single-result implies --only-test
opts.only_test = opts.single_result
if opts.only_test:
# --only-test can either point to a particular test or a directory.
# Therefore, test_suite_root + opts.only_test or
# test_suite_root + dirname(opts.only_test) must be a directory.
path = os.path.join(self.opts.test_suite_root, opts.only_test)
parent_path = os.path.dirname(path)
if os.path.isdir(path):
opts.only_test = (opts.only_test, None)
elif os.path.isdir(parent_path):
opts.only_test = (os.path.dirname(opts.only_test),
os.path.basename(opts.only_test))
else:
parser.error(
"--only-test argument not understood (must be a " +
" test or directory name)")
if opts.single_result and not opts.only_test[1]:
parser.error(
"--single-result must be given a single test name, not a " +
"directory name")
opts.cppflags = ' '.join(opts.cppflags)
opts.cflags = ' '.join(opts.cflags)
opts.cxxflags = ' '.join(opts.cxxflags)
if opts.diagnose:
if not opts.only_test:
parser.error("--diagnose requires --only-test")
self.start_time = timestamp()
# Work out where to put our build stuff
if self.opts.timestamp_build:
ts = self.start_time.replace(' ', '_').replace(':', '-')
build_dir_name = "test-%s" % ts
else:
build_dir_name = "build"
basedir = os.path.join(self.opts.sandbox_path, build_dir_name)
self._base_path = basedir
# We don't support compiling without testing as we can't get compile-
# time numbers from LIT without running the tests.
if opts.compile_multisample > opts.exec_multisample:
note("Increasing number of execution samples to %d" %
opts.compile_multisample)
opts.exec_multisample = opts.compile_multisample
if opts.auto_name:
# Construct the nickname from a few key parameters.
cc_info = self._get_cc_info()
cc_nick = '%s_%s' % (cc_info['cc_name'], cc_info['cc_build'])
self.nick += "__%s__%s" % (cc_nick,
cc_info['cc_target'].split('-')[0])
note('Using nickname: %r' % self.nick)
# If we are doing diagnostics, skip the usual run and do them now.
if opts.diagnose:
return self.diagnose()
# Now do the actual run.
reports = []
for i in range(max(opts.exec_multisample, opts.compile_multisample)):
c = i < opts.compile_multisample
e = i < opts.exec_multisample
reports.append(self.run(self.nick, compile=c, test=e))
report = self._create_merged_report(reports)
# Write the report out so it can be read by the submission tool.
report_path = os.path.join(self._base_path, 'report.json')
with open(report_path, 'w') as fd:
fd.write(report.render())
return self.submit(report_path, self.opts, commit=True)
def run_test(self, opts):
if self.opts.cc is not None:
self.opts.cc = resolve_command_path(self.opts.cc)
if not lnt.testing.util.compilers.is_valid(self.opts.cc):
self._fatal('--cc does not point to a valid executable.')
# If there was no --cxx given, attempt to infer it from the --cc.
if self.opts.cxx is None:
self.opts.cxx = \
lnt.testing.util.compilers.infer_cxx_compiler(self.opts.cc)
if self.opts.cxx is not None:
logger.info("Inferred C++ compiler under test as: %r"
% (self.opts.cxx,))
else:
self._fatal("unable to infer --cxx - set it manually.")
else:
self.opts.cxx = resolve_command_path(self.opts.cxx)
if not os.path.exists(self.opts.cxx):
self._fatal("invalid --cxx argument %r, does not exist"
% (self.opts.cxx))
if opts.test_suite_root is None:
self._fatal('--test-suite is required')
if not os.path.exists(opts.test_suite_root):
self._fatal("invalid --test-suite argument, does not exist: %r" % (
opts.test_suite_root))
opts.test_suite_root = os.path.abspath(opts.test_suite_root)
if opts.test_suite_externals:
if not os.path.exists(opts.test_suite_externals):
self._fatal(
"invalid --test-externals argument, does not exist: %r" % (
opts.test_suite_externals,))
opts.test_suite_externals = os.path.abspath(
opts.test_suite_externals)
opts.cmake = resolve_command_path(opts.cmake)
if not isexecfile(opts.cmake):
self._fatal("CMake tool not found (looked for %s)" % opts.cmake)
opts.make = resolve_command_path(opts.make)
if not isexecfile(opts.make):
self._fatal("Make tool not found (looked for %s)" % opts.make)
opts.lit = resolve_command_path(opts.lit)
if not isexecfile(opts.lit):
self._fatal("LIT tool not found (looked for %s)" % opts.lit)
if opts.run_under:
split = shlex.split(opts.run_under)
split[0] = resolve_command_path(split[0])
if not isexecfile(split[0]):
self._fatal("Run under wrapper not found (looked for %s)" %
opts.run_under)
if opts.single_result:
# --single-result implies --only-test
opts.only_test = opts.single_result
if opts.only_test:
# --only-test can either point to a particular test or a directory.
# Therefore, test_suite_root + opts.only_test or
# test_suite_root + dirname(opts.only_test) must be a directory.
path = os.path.join(self.opts.test_suite_root, opts.only_test)
parent_path = os.path.dirname(path)
if os.path.isdir(path):
opts.only_test = (opts.only_test, None)
elif os.path.isdir(parent_path):
opts.only_test = (os.path.dirname(opts.only_test),
os.path.basename(opts.only_test))
else:
self._fatal("--only-test argument not understood (must be a " +
" test or directory name)")
if opts.single_result and not opts.only_test[1]:
self._fatal("--single-result must be given a single test name, "
"not a directory name")
opts.cppflags = ' '.join(opts.cppflags)
opts.cflags = ' '.join(opts.cflags)
opts.cxxflags = ' '.join(opts.cxxflags)
if opts.diagnose:
if not opts.only_test:
self._fatal("--diagnose requires --only-test")
self.start_time = timestamp()
# Work out where to put our build stuff
if self.opts.timestamp_build:
ts = self.start_time.replace(' ', '_').replace(':', '-')
build_dir_name = "test-%s" % ts
else:
build_dir_name = "build"
basedir = os.path.join(self.opts.sandbox_path, build_dir_name)
self._base_path = basedir
cmakecache = os.path.join(self._base_path, 'CMakeCache.txt')
self.configured = not self.opts.run_configure and \
os.path.exists(cmakecache)
# If we are doing diagnostics, skip the usual run and do them now.
if opts.diagnose:
return self.diagnose()
# configure, so we can extract toolchain information from the cmake
# output.
self._configure_if_needed()
# Verify that we can actually find a compiler before continuing
cmake_vars = self._extract_cmake_vars_from_cache()
if "CMAKE_C_COMPILER" not in cmake_vars or \
not os.path.exists(cmake_vars["CMAKE_C_COMPILER"]):
self._fatal(
"Couldn't find C compiler (%s). Maybe you should specify --cc?"
% cmake_vars.get("CMAKE_C_COMPILER"))
# We don't support compiling without testing as we can't get compile-
# time numbers from LIT without running the tests.
if opts.compile_multisample > opts.exec_multisample:
logger.info("Increasing number of execution samples to %d" %
opts.compile_multisample)
opts.exec_multisample = opts.compile_multisample
if opts.auto_name:
# Construct the nickname from a few key parameters.
cc_info = self._get_cc_info(cmake_vars)
cc_nick = '%s_%s' % (cc_info['cc_name'], cc_info['cc_build'])
opts.label += "__%s__%s" %\
(cc_nick, cc_info['cc_target'].split('-')[0])
logger.info('Using nickname: %r' % opts.label)
# When we can't detect the clang version we use 0 instead. That
# is a horrible failure mode because all of our data ends up going
# to order 0. The user needs to give an order if we can't detect!
if opts.run_order is None:
cc_info = self._get_cc_info(cmake_vars)
if cc_info['inferred_run_order'] == 0:
fatal("Cannot detect compiler version. Specify --run-order"
" to manually define it.")
# Now do the actual run.
reports = []
json_reports = []
for i in range(max(opts.exec_multisample, opts.compile_multisample)):
c = i < opts.compile_multisample
e = i < opts.exec_multisample
# only gather perf profiles on a single run.
p = i == 0 and self.opts.use_perf in ('profile', 'all')
run_report, json_data = self.run(cmake_vars, compile=c, test=e,
profile=p)
reports.append(run_report)
json_reports.append(json_data)
report = self._create_merged_report(reports)
# Write the report out so it can be read by the submission tool.
report_path = os.path.join(self._base_path, 'report.json')
with open(report_path, 'w') as fd:
fd.write(report.render())
if opts.output:
with open(opts.output, 'w') as fd:
fd.write(report.render())
xml_report_path = os.path.join(self._base_path,
'test-results.xunit.xml')
str_template = _lit_json_to_xunit_xml(json_reports)
with open(xml_report_path, 'w') as fd:
fd.write(str_template)
csv_report_path = os.path.join(self._base_path,
'test-results.csv')
str_template = _lit_json_to_csv(json_reports)
with open(csv_report_path, 'w') as fd:
fd.write(str_template)
return self.submit(report_path, self.opts, 'nts')
def run_test(self, name, args):
global opts
parser = OptionParser(
("%(name)s [options] [<output file>]\n" + usage_info) % locals())
parser.add_option("-s",
"--sandbox",
dest="sandbox_path",
help="Parent directory to build and run tests in",
type=str,
default=None,
metavar="PATH")
group = OptionGroup(parser, "Test Options")
group.add_option("",
"--no-timestamp",
dest="timestamp_build",
help="Don't timestamp build directory (for testing)",
action="store_false",
default=True)
group.add_option("",
"--cc",
dest="cc",
type='str',
help="Path to the compiler under test",
action="store",
default=None)
group.add_option("",
"--cxx",
dest="cxx",
help="Path to the C++ compiler to test",
type=str,
default=None)
group.add_option(
"",
"--ld",
dest="ld",
help="Path to the c linker to use. (Xcode Distinction)",
type=str,
default=None)
group.add_option(
"",
"--ldxx",
dest="ldxx",
help="Path to the cxx linker to use. (Xcode Distinction)",
type=str,
default=None)
group.add_option("",
"--test-externals",
dest="test_suite_externals",
help="Path to the LLVM test-suite externals",
type=str,
default=None,
metavar="PATH")
group.add_option("",
"--machine-param",
dest="machine_parameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the machine parameters",
type=str,
action="append",
default=[])
group.add_option("",
"--run-param",
dest="run_parameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the run parameters",
type=str,
action="append",
default=[])
group.add_option("",
"--run-order",
dest="run_order",
metavar="STR",
help="String to use to identify and order this run",
action="store",
type=str,
default=None)
group.add_option(
"",
"--test-subdir",
dest="test_subdir",
help="Subdirectory of test external dir to look for tests in.",
type=str,
default="lnt-compile-suite-src")
parser.add_option_group(group)
group = OptionGroup(parser, "Test Selection")
group.add_option("",
"--no-memory-profiling",
dest="memory_profiling",
help="Disable memory profiling",
action="store_false",
default=True)
group.add_option("",
"--multisample",
dest="run_count",
metavar="N",
help="Accumulate test data from multiple runs",
action="store",
type=int,
default=3)
group.add_option("",
"--min-sample-time",
dest="min_sample_time",
help="Ensure all tests run for at least N seconds",
metavar="N",
action="store",
type=float,
default=.5)
group.add_option("",
"--save-temps",
dest="save_temps",
help="Save temporary build output files",
action="store_true",
default=False)
group.add_option(
"",
"--show-tests",
dest="show_tests",
help="Only list the availables tests that will be run",
action="store_true",
default=False)
group.add_option("",
"--test",
dest="tests",
metavar="NAME",
help="Individual test to run",
action="append",
default=[])
group.add_option("",
"--test-filter",
dest="test_filters",
help="Run tests matching the given pattern",
metavar="REGEXP",
action="append",
default=[])
group.add_option("",
"--flags-to-test",
dest="flags_to_test",
help="Add a set of flags to test (space separated)",
metavar="FLAGLIST",
action="append",
default=[])
group.add_option("",
"--jobs-to-test",
dest="jobs_to_test",
help="Add a job count to test (full builds)",
metavar="NUM",
action="append",
default=[],
type=int)
group.add_option("",
"--config-to-test",
dest="configs_to_test",
help="Add build configuration to test (full builds)",
metavar="NAME",
action="append",
default=[],
choices=('Debug', 'Release'))
parser.add_option_group(group)
group = OptionGroup(parser, "Output Options")
group.add_option("",
"--no-machdep-info",
dest="use_machdep_info",
help=("Don't put machine (instance) dependent "
"variables in machine info"),
action="store_false",
default=True)
group.add_option("",
"--machine-name",
dest="machine_name",
type='str',
help="Machine name to use in submission [%default]",
action="store",
default=platform.uname()[1])
group.add_option(
"",
"--submit",
dest="submit_url",
metavar="URLORPATH",
help=("autosubmit the test result to the given server "
"(or local instance) [%default]"),
type=str,
default=None)
group.add_option(
"",
"--commit",
dest="commit",
help=("whether the autosubmit result should be committed "
"[%default]"),
type=int,
default=True)
group.add_option(
"",
"--output",
dest="output",
metavar="PATH",
help="write raw report data to PATH (or stdout if '-')",
action="store",
default=None)
group.add_option("-v",
"--verbose",
dest="verbose",
help="show verbose test results",
action="store_true",
default=False)
parser.add_option_group(group)
opts, args = parser.parse_args(args)
if len(args) != 0:
parser.error("invalid number of arguments")
if opts.cc is None:
parser.error("You must specify a --cc argument.")
# Resolve the cc_under_test path.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
parser.error('--cc does not point to a valid executable.')
# Attempt to infer the cxx compiler if not given.
if opts.cc and opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
note("inferred C++ compiler under test as: %r" % (opts.cxx, ))
# Validate options.
if opts.cc is None:
parser.error('--cc is required')
if opts.cxx is None:
parser.error('--cxx is required (and could not be inferred)')
if opts.sandbox_path is None:
parser.error('--sandbox is required')
if opts.test_suite_externals is None:
parser.error("--test-externals option is required")
# Force the CC and CXX variables to be absolute paths.
cc_abs = os.path.abspath(commands.which(opts.cc))
cxx_abs = os.path.abspath(commands.which(opts.cxx))
if not os.path.exists(cc_abs):
parser.error("unable to determine absolute path for --cc: %r" %
(opts.cc, ))
if not os.path.exists(cxx_abs):
parser.error("unable to determine absolute path for --cc: %r" %
(opts.cc, ))
opts.cc = cc_abs
opts.cxx = cxx_abs
# If no ld was set, set ld to opts.cc
if opts.ld is None:
opts.ld = opts.cc
# If no ldxx was set, set ldxx to opts.cxx
if opts.ldxx is None:
opts.ldxx = opts.cxx
# Set up the sandbox.
global g_output_dir
if not os.path.exists(opts.sandbox_path):
print >> sys.stderr, "%s: creating sandbox: %r" % (
timestamp(), opts.sandbox_path)
os.mkdir(opts.sandbox_path)
if opts.timestamp_build:
report_name = "test-%s" % (timestamp().replace(' ', '_').replace(
':', '-'))
else:
report_name = "build"
g_output_dir = os.path.join(os.path.abspath(opts.sandbox_path),
report_name)
try:
os.mkdir(g_output_dir)
except OSError, e:
if e.errno == errno.EEXIST:
parser.error("sandbox output directory %r already exists!" %
(g_output_dir, ))
else:
raise
def log(self, message, ts=None):
if not ts:
ts = timestamp()
print('%s: %s' % (ts, message), file=sys.stderr)
