def get_source_version(path):
"""get_source_version(path) -> str or None
Given the path to a revision controlled source tree, return a revision
number, hash, etc. which identifies the source version.
"""
if os.path.exists(os.path.join(path, ".svn")):
return commands.capture(
['/bin/sh', '-c', 'cd "%s" && svnversion' % path]).strip()
elif os.path.exists(os.path.join(path, ".git", "svn")):
# git-svn is pitifully slow, extract the revision manually.
res = commands.capture(
['/bin/sh', '-c', ('cd "%s" && '
'git log -1') % path]).strip()
last_line = res.split("\n")[-1]
m = _git_svn_id_re.match(last_line)
if not m:
logger.warning("unable to understand git svn log: %r" % res)
return
return m.group(1)
elif os.path.exists(os.path.join(path, ".git")):
return commands.capture([
'/bin/sh', '-c', ('cd "%s" && '
'git log -1 --pretty=format:%%H') % path
]).strip()
def get_source_version(path):
"""get_source_version(path) -> str or None
Given the path to a revision controlled source tree, return a revision
number, hash, etc. which identifies the source version.
"""
if os.path.exists(os.path.join(path, ".svn")):
return commands.capture(['/bin/sh', '-c',
'cd "%s" && svnversion' % path]).strip()
elif os.path.exists(os.path.join(path, ".git", "svn")):
# git-svn is pitifully slow, extract the revision manually.
res = commands.capture(['/bin/sh', '-c',
('cd "%s" && '
'git log -1') % path]
).strip()
last_line = res.split("\n")[-1]
m = _git_svn_id_re.match(last_line)
if not m:
commands.warning("unable to understand git svn log: %r" % res)
return
return m.group(1)
elif os.path.exists(os.path.join(path, ".git")):
return commands.capture(['/bin/sh', '-c',
('cd "%s" && '
'git log -1 --pretty=format:%%H') % path]
).strip()
def infer_cxx_compiler(cc_path):
# If this is obviously a compiler name, then try replacing with the '++'
# name.
name = os.path.basename(cc_path)
if 'clang' in name:
expected_cxx_name = 'clang++'
cxx_name = name.replace('clang', expected_cxx_name)
elif 'gcc' in name:
expected_cxx_name = 'g++'
cxx_name = name.replace('gcc', expected_cxx_name)
elif 'icc' in name:
expected_cxx_name = 'icpc'
cxx_name = name.replace('icc', expected_cxx_name)
else:
# We have no idea, give up.
return None
# Check if the compiler exists at that path.
cxx_path = os.path.join(os.path.dirname(cc_path), cxx_name)
if os.path.exists(cxx_path):
return cxx_path
# Otherwise, try to let the compiler itself tell us what the '++' version
# would be. This is useful when the compiler under test is a symlink to the
# real compiler.
cxx_path = capture([cc_path,
'-print-prog-name=%s' % expected_cxx_name]).strip()
if os.path.exists(cxx_path):
return cxx_path
def _get_mac_addresses():
lines = capture(['ifconfig']).strip()
current_ifc = None
for ln in lines.split('\n'):
if ln.startswith('\t'):
if current_ifc is None:
fatal('unexpected ifconfig output')
if ln.startswith('\tether '):
yield current_ifc, ln[len('\tether '):].strip()
else:
current_ifc, = re.match(r'([A-Za-z0-9]*): .*', ln).groups()
def _get_mac_addresses():
lines = capture(['ifconfig']).strip()
current_ifc = None
for ln in lines.split('\n'):
if ln.startswith('\t'):
if current_ifc is None:
fatal('unexpected ifconfig output')
if ln.startswith('\tether '):
yield current_ifc, ln[len('\tether '):].strip()
else:
current_ifc, = re.match(r'([A-Za-z0-9]*): .*', ln).groups()
def get_machine_information(use_machine_dependent_info=False):
machine_info = {}
run_info = {}
info_targets = {
'machdep': (run_info, machine_info)[use_machine_dependent_info],
'machine': machine_info,
'run': run_info
}
for name, target in sysctl_info_table:
info_targets[target][name] = capture(['sysctl', '-n', name],
include_stderr=True).strip()
for ifc, addr in _get_mac_addresses():
# Ignore virtual machine mac addresses.
if ifc.startswith('vmnet'):
continue
info_targets['machdep']['mac_addr.%s' % ifc] = addr
return machine_info, run_info
def get_machine_information(use_machine_dependent_info=False):
machine_info = {}
run_info = {}
info_targets = {
'machdep': (run_info, machine_info)[use_machine_dependent_info],
'machine': machine_info,
'run': run_info,
}
for name, target in sysctl_info_table:
info_targets[target][name] = capture(['sysctl', '-n', name],
include_stderr=True).strip()
for ifc, addr in _get_mac_addresses():
# Ignore virtual machine mac addresses.
if ifc.startswith('vmnet'):
continue
info_targets['machdep']['mac_addr.%s' % ifc] = addr
return machine_info, run_info
class CompileTest(builtintest.BuiltinTest):
def describe(self):
return 'Single file compile-time performance testing'
def run_test(self, name, args):
global opts
parser = OptionParser(
("%(name)s [options] [<output file>]\n" + usage_info) % locals())
parser.add_option("-s",
"--sandbox",
dest="sandbox_path",
help="Parent directory to build and run tests in",
type=str,
default=None,
metavar="PATH")
group = OptionGroup(parser, "Test Options")
group.add_option("",
"--no-timestamp",
dest="timestamp_build",
help="Don't timestamp build directory (for testing)",
action="store_false",
default=True)
group.add_option("",
"--cc",
dest="cc",
type='str',
help="Path to the compiler under test",
action="store",
default=None)
group.add_option("",
"--cxx",
dest="cxx",
help="Path to the C++ compiler to test",
type=str,
default=None)
group.add_option(
"",
"--ld",
dest="ld",
help="Path to the c linker to use. (Xcode Distinction)",
type=str,
default=None)
group.add_option(
"",
"--ldxx",
dest="ldxx",
help="Path to the cxx linker to use. (Xcode Distinction)",
type=str,
default=None)
group.add_option("",
"--test-externals",
dest="test_suite_externals",
help="Path to the LLVM test-suite externals",
type=str,
default=None,
metavar="PATH")
group.add_option("",
"--machine-param",
dest="machine_parameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the machine parameters",
type=str,
action="append",
default=[])
group.add_option("",
"--run-param",
dest="run_parameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the run parameters",
type=str,
action="append",
default=[])
group.add_option("",
"--run-order",
dest="run_order",
metavar="STR",
help="String to use to identify and order this run",
action="store",
type=str,
default=None)
group.add_option(
"",
"--test-subdir",
dest="test_subdir",
help="Subdirectory of test external dir to look for tests in.",
type=str,
default="lnt-compile-suite-src")
parser.add_option_group(group)
group = OptionGroup(parser, "Test Selection")
group.add_option("",
"--no-memory-profiling",
dest="memory_profiling",
help="Disable memory profiling",
action="store_false",
default=True)
group.add_option("",
"--multisample",
dest="run_count",
metavar="N",
help="Accumulate test data from multiple runs",
action="store",
type=int,
default=3)
group.add_option("",
"--min-sample-time",
dest="min_sample_time",
help="Ensure all tests run for at least N seconds",
metavar="N",
action="store",
type=float,
default=.5)
group.add_option("",
"--save-temps",
dest="save_temps",
help="Save temporary build output files",
action="store_true",
default=False)
group.add_option(
"",
"--show-tests",
dest="show_tests",
help="Only list the availables tests that will be run",
action="store_true",
default=False)
group.add_option("",
"--test",
dest="tests",
metavar="NAME",
help="Individual test to run",
action="append",
default=[])
group.add_option("",
"--test-filter",
dest="test_filters",
help="Run tests matching the given pattern",
metavar="REGEXP",
action="append",
default=[])
group.add_option("",
"--flags-to-test",
dest="flags_to_test",
help="Add a set of flags to test (space separated)",
metavar="FLAGLIST",
action="append",
default=[])
group.add_option("",
"--jobs-to-test",
dest="jobs_to_test",
help="Add a job count to test (full builds)",
metavar="NUM",
action="append",
default=[],
type=int)
group.add_option("",
"--config-to-test",
dest="configs_to_test",
help="Add build configuration to test (full builds)",
metavar="NAME",
action="append",
default=[],
choices=('Debug', 'Release'))
parser.add_option_group(group)
group = OptionGroup(parser, "Output Options")
group.add_option("",
"--no-machdep-info",
dest="use_machdep_info",
help=("Don't put machine (instance) dependent "
"variables in machine info"),
action="store_false",
default=True)
group.add_option("",
"--machine-name",
dest="machine_name",
type='str',
help="Machine name to use in submission [%default]",
action="store",
default=platform.uname()[1])
group.add_option(
"",
"--submit",
dest="submit_url",
metavar="URLORPATH",
help=("autosubmit the test result to the given server "
"(or local instance) [%default]"),
type=str,
default=None)
group.add_option(
"",
"--commit",
dest="commit",
help=("whether the autosubmit result should be committed "
"[%default]"),
type=int,
default=True)
group.add_option(
"",
"--output",
dest="output",
metavar="PATH",
help="write raw report data to PATH (or stdout if '-')",
action="store",
default=None)
group.add_option("-v",
"--verbose",
dest="verbose",
help="show verbose test results",
action="store_true",
default=False)
parser.add_option_group(group)
opts, args = parser.parse_args(args)
if len(args) != 0:
parser.error("invalid number of arguments")
if opts.cc is None:
parser.error("You must specify a --cc argument.")
# Resolve the cc_under_test path.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
parser.error('--cc does not point to a valid executable.')
# Attempt to infer the cxx compiler if not given.
if opts.cc and opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
note("inferred C++ compiler under test as: %r" % (opts.cxx, ))
# Validate options.
if opts.cc is None:
parser.error('--cc is required')
if opts.cxx is None:
parser.error('--cxx is required (and could not be inferred)')
if opts.sandbox_path is None:
parser.error('--sandbox is required')
if opts.test_suite_externals is None:
parser.error("--test-externals option is required")
# Force the CC and CXX variables to be absolute paths.
cc_abs = os.path.abspath(commands.which(opts.cc))
cxx_abs = os.path.abspath(commands.which(opts.cxx))
if not os.path.exists(cc_abs):
parser.error("unable to determine absolute path for --cc: %r" %
(opts.cc, ))
if not os.path.exists(cxx_abs):
parser.error("unable to determine absolute path for --cc: %r" %
(opts.cc, ))
opts.cc = cc_abs
opts.cxx = cxx_abs
# If no ld was set, set ld to opts.cc
if opts.ld is None:
opts.ld = opts.cc
# If no ldxx was set, set ldxx to opts.cxx
if opts.ldxx is None:
opts.ldxx = opts.cxx
# Set up the sandbox.
global g_output_dir
if not os.path.exists(opts.sandbox_path):
print >> sys.stderr, "%s: creating sandbox: %r" % (
timestamp(), opts.sandbox_path)
os.mkdir(opts.sandbox_path)
if opts.timestamp_build:
report_name = "test-%s" % (timestamp().replace(' ', '_').replace(
':', '-'))
else:
report_name = "build"
g_output_dir = os.path.join(os.path.abspath(opts.sandbox_path),
report_name)
try:
os.mkdir(g_output_dir)
except OSError, e:
if e.errno == errno.EEXIST:
parser.error("sandbox output directory %r already exists!" %
(g_output_dir, ))
else:
raise
# Setup log file
global g_log
def setup_log(output_dir):
def stderr_log_handler():
h = logging.StreamHandler()
f = logging.Formatter(
"%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
def file_log_handler(path):
h = logging.FileHandler(path, mode='w')
f = logging.Formatter(
"%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
l = logging.Logger('compile_test')
l.setLevel(logging.INFO)
l.addHandler(file_log_handler(os.path.join(output_dir,
'test.log')))
l.addHandler(stderr_log_handler())
return l
g_log = setup_log(g_output_dir)
# Collect machine and run information.
machine_info, run_info = machineinfo.get_machine_information(
opts.use_machdep_info)
# FIXME: Include information on test source versions.
#
# FIXME: Get more machine information? Cocoa.h hash, for example.
for name, cmd in (('sys_cc_version', ('/usr/bin/gcc', '-v')),
('sys_as_version', ('/usr/bin/as', '-v',
'/dev/null')),
('sys_ld_version', ('/usr/bin/ld', '-v')),
('sys_xcodebuild', ('xcodebuild', '-version'))):
run_info[name] = commands.capture(cmd, include_stderr=True).strip()
# Set command line machine and run information.
for info, params in ((machine_info, opts.machine_parameters),
(run_info, opts.run_parameters)):
for entry in params:
if '=' not in entry:
name, value = entry, ''
else:
name, value = entry.split('=', 1)
info[name] = value
# Set user variables.
variables = {}
variables['cc'] = opts.cc
variables['run_count'] = opts.run_count
# Get compiler info.
cc_info = lnt.testing.util.compilers.get_cc_info(variables['cc'])
variables.update(cc_info)
# Set the run order from the user, if given.
if opts.run_order is not None:
variables['run_order'] = opts.run_order
else:
# Otherwise, use the inferred run order.
variables['run_order'] = cc_info['inferred_run_order']
note("inferred run order to be: %r" % (variables['run_order'], ))
if opts.verbose:
format = pprint.pformat(variables)
msg = '\n\t'.join(['using variables:'] + format.splitlines())
note(msg)
format = pprint.pformat(machine_info)
msg = '\n\t'.join(['using machine info:'] + format.splitlines())
note(msg)
format = pprint.pformat(run_info)
msg = '\n\t'.join(['using run info:'] + format.splitlines())
note(msg)
# Compute the set of flags to test.
if not opts.flags_to_test:
flags_to_test = [('-O0', ), (
'-O0',
'-g',
), ('-Os', '-g'), ('-O3', )]
else:
flags_to_test = [
string.split(' ') for string in opts.flags_to_test
]
# Compute the set of job counts to use in full build tests.
if not opts.jobs_to_test:
jobs_to_test = [1, 2, 4, 8]
else:
jobs_to_test = opts.jobs_to_test
# Compute the build configurations to test.
if not opts.configs_to_test:
configs_to_test = ['Debug', 'Release']
else:
configs_to_test = opts.configs_to_test
# Compute the list of all tests.
all_tests = list(
get_tests(opts.test_suite_externals, opts.test_subdir,
flags_to_test, jobs_to_test, configs_to_test))
# Show the tests, if requested.
if opts.show_tests:
print >> sys.stderr, 'Available Tests'
for name in sorted(set(name for name, _ in all_tests)):
print >> sys.stderr, ' %s' % (name, )
print
raise SystemExit
# Find the tests to run.
if not opts.tests and not opts.test_filters:
tests_to_run = list(all_tests)
else:
all_test_names = set(test[0] for test in all_tests)
# Validate the test names.
requested_tests = set(opts.tests)
missing_tests = requested_tests - all_test_names
if missing_tests:
parser.error(("invalid test names %s, use --show-tests to "
"see available tests") %
(", ".join(map(repr, missing_tests)), ))
# Validate the test filters.
test_filters = [
re.compile(pattern) for pattern in opts.test_filters
]
# Form the list of tests.
tests_to_run = [
test for test in all_tests if
(test[0] in requested_tests or
[True for filter in test_filters if filter.search(test[0])])
]
if not tests_to_run:
parser.error(
"no tests requested (invalid --test or --test-filter options)!"
)
# Ensure output directory is available.
if not os.path.exists(g_output_dir):
os.mkdir(g_output_dir)
# Execute the run.
run_info.update(variables)
run_info['tag'] = tag = 'compile'
testsamples = []
start_time = datetime.utcnow()
g_log.info('run started')
g_log.info('using CC: %r' % opts.cc)
g_log.info('using CXX: %r' % opts.cxx)
for basename, test_fn in tests_to_run:
for success, name, samples in test_fn(basename, run_info,
variables):
g_log.info('collected samples: %r' % name)
num_samples = len(samples)
if num_samples:
samples_median = '%.4f' % (stats.median(samples), )
samples_mad = '%.4f' % (
stats.median_absolute_deviation(samples), )
else:
samples_median = samples_mad = 'N/A'
g_log.info('N=%d, median=%s, MAD=%s' %
(num_samples, samples_median, samples_mad))
test_name = '%s.%s' % (tag, name)
if not success:
testsamples.append(
lnt.testing.TestSamples(test_name + '.status',
[lnt.testing.FAIL]))
if samples:
testsamples.append(
lnt.testing.TestSamples(test_name, samples))
end_time = datetime.utcnow()
g_log.info('run complete')
# Package up the report.
machine = lnt.testing.Machine(opts.machine_name, machine_info)
run = lnt.testing.Run(start_time, end_time, info=run_info)
# Write out the report.
lnt_report_path = os.path.join(g_output_dir, 'report.json')
report = lnt.testing.Report(machine, run, testsamples)
# Save report to disk for submission.
self.print_report(report, lnt_report_path)
# Then, also print to screen if requested.
if opts.output is not None:
self.print_report(report, opts.output)
server_report = self.submit(lnt_report_path, opts)
return server_report
def test_cc_command(base_name,
run_info,
variables,
input,
output,
flags,
extra_flags,
has_output=True,
ignore_stderr=False):
name = '%s/(%s)' % (
base_name,
' '.join(flags),
)
input = get_input_path(opts, input)
output = get_output_path(output)
cmd = [variables.get('cc')]
cmd.extend(extra_flags)
cmd.append(input)
cmd.extend(flags)
# Inhibit all warnings, we don't want to count the time to generate them
# against newer compilers which have added (presumably good) warnings.
cmd.append('-w')
# Do a memory profiling run, if requested.
#
# FIXME: Doing this as a separate step seems silly. We shouldn't do any
# extra run just to get the memory statistics.
if opts.memory_profiling:
# Find the cc1 command, which we use to do memory profiling. To do this
# we execute the compiler with '-###' to figure out what it wants to do.
cc_output = commands.capture(cmd + ['-o', '/dev/null', '-###'],
include_stderr=True).strip()
cc_commands = []
for ln in cc_output.split('\n'):
# Filter out known garbage.
if (ln == 'Using built-in specs.'
or ln.startswith('Configured with:')
or ln.startswith('Target:')
or ln.startswith('Thread model:')
or ln.startswith('InstalledDir:') or ' version ' in ln):
continue
cc_commands.append(ln)
if len(cc_commands) != 1:
fatal('unable to determine cc1 command: %r' % cc_output)
cc1_cmd = shlex.split(cc_commands[0])
for res in get_runN_test_data(name,
variables,
cc1_cmd,
ignore_stderr=ignore_stderr,
sample_mem=True,
only_mem=True):
yield res
commands.rm_f(output)
for res in get_runN_test_data(name,
variables,
cmd + ['-o', output],
ignore_stderr=ignore_stderr):
yield res
# If the command has output, track its size.
if has_output:
tname = '%s.size' % (name, )
success = False
samples = []
try:
stat = os.stat(output)
success = True
# For now, the way the software is set up things are going to get
# confused if we don't report the same number of samples as reported
# for other variables. So we just report the size N times.
#
# FIXME: We should resolve this, eventually.
for i in range(variables.get('run_count')):
samples.append(stat.st_size)
except OSError, e:
if e.errno != errno.ENOENT:
raise
yield (success, tname, samples)
def test_cc_command(base_name, run_info, variables, input, output, flags,
extra_flags, has_output=True, ignore_stderr=False):
name = '%s/(%s)' % (base_name,' '.join(flags),)
input = get_input_path(opts, input)
output = get_output_path(output)
cmd = [variables.get('cc')]
cmd.extend(extra_flags)
cmd.append(input)
cmd.extend(flags)
# Inhibit all warnings, we don't want to count the time to generate them
# against newer compilers which have added (presumably good) warnings.
cmd.append('-w')
# Do a memory profiling run, if requested.
#
# FIXME: Doing this as a separate step seems silly. We shouldn't do any
# extra run just to get the memory statistics.
if opts.memory_profiling:
# Find the cc1 command, which we use to do memory profiling. To do this
# we execute the compiler with '-###' to figure out what it wants to do.
cc_output = commands.capture(cmd + ['-o','/dev/null','-###'],
include_stderr=True).strip()
cc_commands = []
for ln in cc_output.split('\n'):
# Filter out known garbage.
if (ln == 'Using built-in specs.' or
ln.startswith('Configured with:') or
ln.startswith('Target:') or
ln.startswith('Thread model:') or
ln.startswith('InstalledDir:') or
' version ' in ln):
continue
cc_commands.append(ln)
if len(cc_commands) != 1:
fatal('unable to determine cc1 command: %r' % cc_output)
cc1_cmd = shlex.split(cc_commands[0])
for res in get_runN_test_data(name, variables, cc1_cmd,
ignore_stderr=ignore_stderr,
sample_mem=True, only_mem=True):
yield res
commands.rm_f(output)
for res in get_runN_test_data(name, variables, cmd + ['-o',output],
ignore_stderr=ignore_stderr):
yield res
# If the command has output, track its size.
if has_output:
tname = '%s.size' % (name,)
success = False
samples = []
try:
stat = os.stat(output)
success = True
# For now, the way the software is set up things are going to get
# confused if we don't report the same number of samples as reported
# for other variables. So we just report the size N times.
#
# FIXME: We should resolve this, eventually.
for i in range(variables.get('run_count')):
samples.append(stat.st_size)
except OSError,e:
if e.errno != errno.ENOENT:
raise
yield (success, tname, samples)
def run_test(self, opts):
# Resolve the cc_under_test path.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
self._fatal('--cc does not point to a valid executable.')
# Attempt to infer the cxx compiler if not given.
if opts.cc and opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
logger.info("inferred C++ compiler under test as: %r" %
(opts.cxx, ))
if opts.cxx is None:
self._fatal('--cxx is required (and could not be inferred)')
# Force the CC and CXX variables to be absolute paths.
cc_abs = os.path.abspath(commands.which(opts.cc))
cxx_abs = os.path.abspath(commands.which(opts.cxx))
if not os.path.exists(cc_abs):
self._fatal("unable to determine absolute path for --cc: %r" %
(opts.cc, ))
if not os.path.exists(cxx_abs):
self._fatal("unable to determine absolute path for --cc: %r" %
(opts.cc, ))
opts.cc = cc_abs
opts.cxx = cxx_abs
# If no ld was set, set ld to opts.cc
if opts.ld is None:
opts.ld = opts.cc
# If no ldxx was set, set ldxx to opts.cxx
if opts.ldxx is None:
opts.ldxx = opts.cxx
# Set up the sandbox.
global g_output_dir
if not os.path.exists(opts.sandbox_path):
print >> sys.stderr, "%s: creating sandbox: %r" % (
timestamp(), opts.sandbox_path)
os.mkdir(opts.sandbox_path)
if opts.timestamp_build:
fmt_timestamp = timestamp().replace(' ', '_').replace(':', '-')
report_name = "test-%s" % (fmt_timestamp)
else:
report_name = "build"
g_output_dir = os.path.join(os.path.abspath(opts.sandbox_path),
report_name)
try:
os.mkdir(g_output_dir)
except OSError as e:
if e.errno == errno.EEXIST:
self._fatal("sandbox output directory %r already exists!" %
(g_output_dir, ))
else:
raise
# Setup log file
global g_log
def setup_log(output_dir):
def stderr_log_handler():
h = logging.StreamHandler()
f = logging.Formatter(
"%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
def file_log_handler(path):
h = logging.FileHandler(path, mode='w')
f = logging.Formatter(
"%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
log = logging.Logger('compile_test')
log.setLevel(logging.INFO)
log.addHandler(
file_log_handler(os.path.join(output_dir, 'test.log')))
log.addHandler(stderr_log_handler())
return log
g_log = setup_log(g_output_dir)
# Collect machine and run information.
machine_info, run_info = machineinfo.get_machine_information(
opts.use_machdep_info)
# FIXME: Include information on test source versions.
#
# FIXME: Get more machine information? Cocoa.h hash, for example.
for name, cmd in (('sys_cc_version', ('/usr/bin/gcc', '-v')),
('sys_as_version', ('/usr/bin/as', '-v',
'/dev/null')),
('sys_ld_version', ('/usr/bin/ld', '-v')),
('sys_xcodebuild', ('xcodebuild', '-version'))):
run_info[name] = commands.capture(cmd, include_stderr=True).strip()
# Set command line machine and run information.
for info, params in ((machine_info, opts.machine_parameters),
(run_info, opts.run_parameters)):
for entry in params:
if '=' not in entry:
name, value = entry, ''
else:
name, value = entry.split('=', 1)
info[name] = value
# Set user variables.
variables = {}
variables['cc'] = opts.cc
variables['run_count'] = opts.run_count
# Get compiler info.
cc_info = lnt.testing.util.compilers.get_cc_info(variables['cc'])
variables.update(cc_info)
# Set the run order from the user, if given.
if opts.run_order is not None:
variables['run_order'] = opts.run_order
else:
# Otherwise, use the inferred run order.
variables['run_order'] = cc_info['inferred_run_order']
logger.info("inferred run order to be: %r" %
(variables['run_order'], ))
if opts.verbose:
format = pprint.pformat(variables)
msg = '\n\t'.join(['using variables:'] + format.splitlines())
logger.info(msg)
format = pprint.pformat(machine_info)
msg = '\n\t'.join(['using machine info:'] + format.splitlines())
logger.info(msg)
format = pprint.pformat(run_info)
msg = '\n\t'.join(['using run info:'] + format.splitlines())
logger.info(msg)
# Compute the set of flags to test.
if not opts.flags_to_test:
flags_to_test = DEFAULT_FLAGS_TO_TEST
else:
flags_to_test = [
string.split(' ') for string in opts.flags_to_test
]
# Compute the set of job counts to use in full build tests.
if not opts.jobs_to_test:
jobs_to_test = [1, 2, 4, 8]
else:
jobs_to_test = opts.jobs_to_test
# Compute the build configurations to test.
if not opts.configs_to_test:
configs_to_test = ['Debug', 'Release']
else:
configs_to_test = opts.configs_to_test
# Compute the list of all tests.
all_tests = list(
get_tests(opts.test_suite_externals, opts.test_subdir,
flags_to_test, jobs_to_test, configs_to_test))
# Show the tests, if requested.
if opts.show_tests:
print >> sys.stderr, 'Available Tests'
for name in sorted(set(name for name, _ in all_tests)):
print >> sys.stderr, ' %s' % (name, )
print
raise SystemExit
# Find the tests to run.
if not opts.tests and not opts.test_filters:
tests_to_run = list(all_tests)
else:
all_test_names = set(test[0] for test in all_tests)
# Validate the test names.
requested_tests = set(opts.tests)
missing_tests = requested_tests - all_test_names
if missing_tests:
self._fatal(("invalid test names %s, use --show-tests to "
"see available tests") %
(", ".join(map(repr, missing_tests)), ))
# Validate the test filters.
test_filters = [
re.compile(pattern) for pattern in opts.test_filters
]
# Form the list of tests.
tests_to_run = [
test for test in all_tests if
(test[0] in requested_tests or
[True for filter in test_filters if filter.search(test[0])])
]
if not tests_to_run:
self._fatal("no tests requested "
"(invalid --test or --test-filter options)!")
# Ensure output directory is available.
if not os.path.exists(g_output_dir):
os.mkdir(g_output_dir)
# Execute the run.
run_info.update(variables)
run_info['tag'] = tag = 'compile'
testsamples = []
start_time = datetime.utcnow()
g_log.info('run started')
g_log.info('using CC: %r' % opts.cc)
g_log.info('using CXX: %r' % opts.cxx)
no_errors = True
for basename, test_fn in tests_to_run:
for success, name, samples in test_fn(basename, run_info,
variables):
g_log.info('collected samples: %r' % name)
num_samples = len(samples)
if num_samples:
samples_median = '%.4f' % (stats.median(samples), )
samples_mad = '%.4f' % (
stats.median_absolute_deviation(samples), )
else:
samples_median = samples_mad = 'N/A'
g_log.info('N=%d, median=%s, MAD=%s' %
(num_samples, samples_median, samples_mad))
test_name = '%s.%s' % (tag, name)
if not success:
testsamples.append(
lnt.testing.TestSamples(test_name + '.status',
[lnt.testing.FAIL]))
no_errors = False
if samples:
testsamples.append(
lnt.testing.TestSamples(test_name, samples))
run_info['no_errors'] = no_errors
end_time = datetime.utcnow()
g_log.info('run complete')
# Package up the report.
machine = lnt.testing.Machine(opts.machine_name, machine_info)
run = lnt.testing.Run(start_time, end_time, info=run_info)
# Write out the report.
lnt_report_path = os.path.join(g_output_dir, 'report.json')
report = lnt.testing.Report(machine, run, testsamples)
# Save report to disk for submission.
self.print_report(report, lnt_report_path)
# Then, also print to screen if requested.
if opts.output is not None:
self.print_report(report, opts.output)
server_report = self.submit(lnt_report_path, opts, ts_name='compile')
return server_report
def get_cc_info(path, cc_flags=[]):
"""get_cc_info(path) -> { ... }
Extract various information on the given compiler and return a dictionary
of the results."""
cc = path
# Interrogate the compiler.
cc_version = capture([cc, '-v', '-E'] + cc_flags +
['-x', 'c', '/dev/null', '-###'],
include_stderr=True).strip()
# Determine the assembler version, as found by the compiler.
cc_as_version = capture([cc, "-c", '-Wa,-v', '-o', '/dev/null'] +
cc_flags + ['-x', 'assembler', '/dev/null'],
include_stderr=True).strip()
if "clang: error: unsupported argument '-v'" in cc_as_version:
cc_as_version = "Clang built in."
# Determine the linker version, as found by the compiler.
cc_ld_version = capture(([cc, "-Wl,-v", "-dynamiclib"]),
include_stderr=True).strip()
# Extract the default target .ll (or assembly, for non-LLVM compilers).
cc_target_assembly = capture([cc, '-S', '-flto', '-o', '-'] + cc_flags +
['-x', 'c', '/dev/null'],
include_stderr=True).strip()
# Extract the compiler's response to -dumpmachine as the target.
cc_target = cc_dumpmachine = capture([cc, '-dumpmachine']).strip()
# Default the target to the response from dumpmachine.
cc_target = cc_dumpmachine
# Parse out the compiler's version line and the path to the "cc1" binary.
cc1_binary = None
version_ln = None
cc_name = cc_version_num = cc_build_string = cc_extra = ""
for ln in cc_version.split('\n'):
if ' version ' in ln:
version_ln = ln
elif 'cc1' in ln or 'clang-cc' in ln:
m = re.match(r' "?([^"]*)"?.*"?-E"?.*', ln)
if not m:
fatal("unable to determine cc1 binary: %r: %r" % (cc, ln))
cc1_binary, = m.groups()
elif "-_Amachine" in ln:
m = re.match(r'([^ ]*) *-.*', ln)
if not m:
fatal("unable to determine cc1 binary: %r: %r" % (cc, ln))
cc1_binary, = m.groups()
if cc1_binary is None:
logger.error("unable to find compiler cc1 binary: %r: %r" %
(cc, cc_version))
if version_ln is None:
logger.error("unable to find compiler version: %r: %r" %
(cc, cc_version))
else:
m = re.match(r'(.*) version ([^ ]*) +(\([^(]*\))(.*)', version_ln)
if m is not None:
cc_name, cc_version_num, cc_build_string, cc_extra = m.groups()
else:
# If that didn't match, try a more basic pattern.
m = re.match(r'(.*) version ([^ ]*)', version_ln)
if m is not None:
cc_name, cc_version_num = m.groups()
else:
logger.error("unable to determine compiler version: %r: %r" %
(cc, version_ln))
cc_name = "unknown"
# Compute normalized compiler name and type. We try to grab source
# revisions, branches, and tags when possible.
cc_norm_name = None
cc_build = None
cc_src_branch = cc_alt_src_branch = None
cc_src_revision = cc_alt_src_revision = None
cc_src_tag = None
llvm_capable = False
cc_extra = cc_extra.strip()
if cc_name == 'icc':
cc_norm_name = 'icc'
cc_build = 'PROD'
cc_src_tag = cc_version_num
elif cc_name == 'gcc' and (cc_extra == '' or
re.match(r' \(dot [0-9]+\)', cc_extra)):
cc_norm_name = 'gcc'
m = re.match(r'\(Apple Inc. build ([0-9]*)\)', cc_build_string)
if m:
cc_build = 'PROD'
cc_src_tag, = m.groups()
else:
logger.error('unable to determine gcc build version: %r' %
cc_build_string)
elif (cc_name in ('clang', 'LLVM', 'Debian clang', 'Apple clang',
'Apple LLVM') and
(cc_extra == '' or 'based on LLVM' in cc_extra or
(cc_extra.startswith('(') and cc_extra.endswith(')')))):
llvm_capable = True
if cc_name == 'Apple clang' or cc_name == 'Apple LLVM':
cc_norm_name = 'apple_clang'
else:
cc_norm_name = 'clang'
m = re.match(r'\(([^ ]*)( ([0-9]+))?\)', cc_build_string)
if m:
cc_src_branch, _, cc_src_revision = m.groups()
# With a CMake build, the branch is not emitted.
if cc_src_branch and not cc_src_revision and \
cc_src_branch.isdigit():
cc_src_revision = cc_src_branch
cc_src_branch = ""
# These show up with git-svn.
if cc_src_branch == '$URL$':
cc_src_branch = ""
else:
# Otherwise, see if we can match a branch and a tag name. That
# could be a git hash.
m = re.match(r'\((.+) ([^ ]+)\)', cc_build_string)
if m:
cc_src_branch, cc_src_revision = m.groups()
else:
logger.error('unable to determine '
'Clang development build info: %r' %
((cc_name, cc_build_string, cc_extra),))
cc_src_branch = ""
m = re.search('clang-([0-9.]*)', cc_src_branch)
if m:
cc_build = 'PROD'
cc_src_tag, = m.groups()
# We sometimes use a tag of 9999 to indicate a dev build.
if cc_src_tag == '9999':
cc_build = 'DEV'
else:
cc_build = 'DEV'
# Newer Clang's can report separate versions for LLVM and Clang. Parse
# the cc_extra text so we can get the maximum SVN version.
if cc_extra.startswith('(') and cc_extra.endswith(')'):
m = re.match(r'\((.+) ([^ ]+)\)', cc_extra)
if m:
cc_alt_src_branch, cc_alt_src_revision = m.groups()
# With a CMake build, the branch is not emitted.
if cc_src_branch and not cc_src_revision and \
cc_src_branch.isdigit():
cc_alt_src_revision = cc_alt_src_branch
cc_alt_src_branch = ""
else:
logger.error('unable to determine '
'Clang development build info: %r' %
((cc_name, cc_build_string, cc_extra), ))
elif cc_name == 'gcc' and 'LLVM build' in cc_extra:
llvm_capable = True
cc_norm_name = 'llvm-gcc'
m = re.match(r' \(LLVM build ([0-9.]+)\)', cc_extra)
if m:
llvm_build, = m.groups()
if llvm_build:
cc_src_tag = llvm_build.strip()
cc_build = 'PROD'
else:
cc_build = 'DEV'
else:
logger.error("unable to determine compiler name: %r" %
((cc_name, cc_build_string),))
if cc_build is None:
logger.error("unable to determine compiler build: %r" % cc_version)
# If LLVM capable, fetch the llvm target instead.
if llvm_capable:
m = re.search('target triple = "(.*)"', cc_target_assembly)
if m:
cc_target, = m.groups()
else:
logger.error("unable to determine LLVM compiler target: %r: %r" %
(cc, cc_target_assembly))
cc_exec_hash = hashlib.sha1()
cc_exec_hash.update(open(cc, 'rb').read())
info = {
'cc_build': cc_build,
'cc_name': cc_norm_name,
'cc_version_number': cc_version_num,
'cc_dumpmachine': cc_dumpmachine,
'cc_target': cc_target,
'cc_version': cc_version,
'cc_exec_hash': cc_exec_hash.hexdigest(),
'cc_as_version': cc_as_version,
'cc_ld_version': cc_ld_version,
'cc_target_assembly': cc_target_assembly,
}
if cc1_binary is not None and os.path.exists(cc1_binary):
cc1_exec_hash = hashlib.sha1()
cc1_exec_hash.update(open(cc1_binary, 'rb').read())
info['cc1_exec_hash'] = cc1_exec_hash.hexdigest()
if cc_src_tag is not None:
info['cc_src_tag'] = cc_src_tag
if cc_src_revision is not None:
info['cc_src_revision'] = cc_src_revision
if cc_src_branch:
info['cc_src_branch'] = cc_src_branch
if cc_alt_src_revision is not None:
info['cc_alt_src_revision'] = cc_alt_src_revision
if cc_alt_src_branch is not None:
info['cc_alt_src_branch'] = cc_alt_src_branch
# Infer the run order from the other things we have computed.
info['inferred_run_order'] = get_inferred_run_order(info)
return info
def run_test(self, opts):
# Resolve the cc_under_test path.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
self._fatal('--cc does not point to a valid executable.')
# Attempt to infer the cxx compiler if not given.
if opts.cc and opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
logger.info("inferred C++ compiler under test as: %r" %
(opts.cxx,))
if opts.cxx is None:
self._fatal('--cxx is required (and could not be inferred)')
# Force the CC and CXX variables to be absolute paths.
cc_abs = os.path.abspath(commands.which(opts.cc))
cxx_abs = os.path.abspath(commands.which(opts.cxx))
if not os.path.exists(cc_abs):
self._fatal("unable to determine absolute path for --cc: %r" % (
opts.cc,))
if not os.path.exists(cxx_abs):
self._fatal("unable to determine absolute path for --cc: %r" % (
opts.cc,))
opts.cc = cc_abs
opts.cxx = cxx_abs
# If no ld was set, set ld to opts.cc
if opts.ld is None:
opts.ld = opts.cc
# If no ldxx was set, set ldxx to opts.cxx
if opts.ldxx is None:
opts.ldxx = opts.cxx
# Set up the sandbox.
global g_output_dir
if not os.path.exists(opts.sandbox_path):
print >>sys.stderr, "%s: creating sandbox: %r" % (
timestamp(), opts.sandbox_path)
os.mkdir(opts.sandbox_path)
if opts.timestamp_build:
fmt_timestamp = timestamp().replace(' ', '_').replace(':', '-')
report_name = "test-%s" % (fmt_timestamp)
else:
report_name = "build"
g_output_dir = os.path.join(os.path.abspath(opts.sandbox_path),
report_name)
try:
os.mkdir(g_output_dir)
except OSError as e:
if e.errno == errno.EEXIST:
self._fatal("sandbox output directory %r already exists!" % (
g_output_dir,))
else:
raise
# Setup log file
global g_log
def setup_log(output_dir):
def stderr_log_handler():
h = logging.StreamHandler()
f = logging.Formatter(
"%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
def file_log_handler(path):
h = logging.FileHandler(path, mode='w')
f = logging.Formatter(
"%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
log = logging.Logger('compile_test')
log.setLevel(logging.INFO)
log.addHandler(file_log_handler(os.path.join(output_dir,
'test.log')))
log.addHandler(stderr_log_handler())
return log
g_log = setup_log(g_output_dir)
# Collect machine and run information.
machine_info, run_info = machineinfo.get_machine_information(
opts.use_machdep_info)
# FIXME: Include information on test source versions.
#
# FIXME: Get more machine information? Cocoa.h hash, for example.
for name, cmd in (('sys_cc_version', ('/usr/bin/gcc', '-v')),
('sys_as_version',
('/usr/bin/as', '-v', '/dev/null')),
('sys_ld_version', ('/usr/bin/ld', '-v')),
('sys_xcodebuild', ('xcodebuild', '-version'))):
run_info[name] = commands.capture(cmd, include_stderr=True).strip()
# Set command line machine and run information.
for info, params in ((machine_info, opts.machine_parameters),
(run_info, opts.run_parameters)):
for entry in params:
if '=' not in entry:
name, value = entry, ''
else:
name, value = entry.split('=', 1)
info[name] = value
# Set user variables.
variables = {}
variables['cc'] = opts.cc
variables['run_count'] = opts.run_count
# Get compiler info.
cc_info = lnt.testing.util.compilers.get_cc_info(variables['cc'])
variables.update(cc_info)
# Set the run order from the user, if given.
if opts.run_order is not None:
variables['run_order'] = opts.run_order
else:
# Otherwise, use the inferred run order.
variables['run_order'] = cc_info['inferred_run_order']
logger.info("inferred run order to be: %r" %
(variables['run_order'],))
if opts.verbose:
format = pprint.pformat(variables)
msg = '\n\t'.join(['using variables:'] + format.splitlines())
logger.info(msg)
format = pprint.pformat(machine_info)
msg = '\n\t'.join(['using machine info:'] + format.splitlines())
logger.info(msg)
format = pprint.pformat(run_info)
msg = '\n\t'.join(['using run info:'] + format.splitlines())
logger.info(msg)
# Compute the set of flags to test.
if not opts.flags_to_test:
flags_to_test = DEFAULT_FLAGS_TO_TEST
else:
flags_to_test = [string.split(' ')
for string in opts.flags_to_test]
# Compute the set of job counts to use in full build tests.
if not opts.jobs_to_test:
jobs_to_test = [1, 2, 4, 8]
else:
jobs_to_test = opts.jobs_to_test
# Compute the build configurations to test.
if not opts.configs_to_test:
configs_to_test = ['Debug', 'Release']
else:
configs_to_test = opts.configs_to_test
# Compute the list of all tests.
all_tests = list(get_tests(opts.test_suite_externals, opts.test_subdir,
flags_to_test, jobs_to_test,
configs_to_test))
# Show the tests, if requested.
if opts.show_tests:
print >>sys.stderr, 'Available Tests'
for name in sorted(set(name for name, _ in all_tests)):
print >>sys.stderr, ' %s' % (name,)
print
raise SystemExit
# Find the tests to run.
if not opts.tests and not opts.test_filters:
tests_to_run = list(all_tests)
else:
all_test_names = set(test[0] for test in all_tests)
# Validate the test names.
requested_tests = set(opts.tests)
missing_tests = requested_tests - all_test_names
if missing_tests:
self._fatal(("invalid test names %s, use --show-tests to "
"see available tests") %
(", ".join(map(repr, missing_tests)), ))
# Validate the test filters.
test_filters = [re.compile(pattern)
for pattern in opts.test_filters]
# Form the list of tests.
tests_to_run = [test
for test in all_tests
if (test[0] in requested_tests or
[True
for filter in test_filters
if filter.search(test[0])])]
if not tests_to_run:
self._fatal("no tests requested "
"(invalid --test or --test-filter options)!")
# Ensure output directory is available.
if not os.path.exists(g_output_dir):
os.mkdir(g_output_dir)
# Execute the run.
run_info.update(variables)
run_info['tag'] = tag = 'compile'
testsamples = []
start_time = datetime.utcnow()
g_log.info('run started')
g_log.info('using CC: %r' % opts.cc)
g_log.info('using CXX: %r' % opts.cxx)
no_errors = True
for basename, test_fn in tests_to_run:
for success, name, samples in test_fn(basename, run_info,
variables):
g_log.info('collected samples: %r' % name)
num_samples = len(samples)
if num_samples:
samples_median = '%.4f' % (stats.median(samples),)
samples_mad = '%.4f' % (
stats.median_absolute_deviation(samples),)
else:
samples_median = samples_mad = 'N/A'
g_log.info('N=%d, median=%s, MAD=%s' % (
num_samples, samples_median, samples_mad))
test_name = '%s.%s' % (tag, name)
if not success:
testsamples.append(lnt.testing.TestSamples(
test_name + '.status', [lnt.testing.FAIL]))
no_errors = False
if samples:
testsamples.append(lnt.testing.TestSamples(
test_name, samples))
run_info['no_errors'] = no_errors
end_time = datetime.utcnow()
g_log.info('run complete')
# Package up the report.
machine = lnt.testing.Machine(opts.machine_name, machine_info)
run = lnt.testing.Run(start_time, end_time, info=run_info)
# Write out the report.
lnt_report_path = os.path.join(g_output_dir, 'report.json')
report = lnt.testing.Report(machine, run, testsamples)
# Save report to disk for submission.
self.print_report(report, lnt_report_path)
# Then, also print to screen if requested.
if opts.output is not None:
self.print_report(report, opts.output)
server_report = self.submit(lnt_report_path, opts, ts_name='compile')
return server_report
def run_test(self, name, args):
global opts
parser = OptionParser(
("%(name)s [options] [<output file>]\n" +
usage_info) % locals())
parser.add_option("-s", "--sandbox", dest="sandbox_path",
help="Parent directory to build and run tests in",
type=str, default=None, metavar="PATH")
group = OptionGroup(parser, "Test Options")
group.add_option("", "--no-timestamp", dest="timestamp_build",
help="Don't timestamp build directory (for testing)",
action="store_false", default=True)
group.add_option("", "--cc", dest="cc", type='str',
help="Path to the compiler under test",
action="store", default=None)
group.add_option("", "--cxx", dest="cxx",
help="Path to the C++ compiler to test",
type=str, default=None)
group.add_option("", "--ld", dest="ld",
help="Path to the c linker to use. (Xcode Distinction)",
type=str, default=None)
group.add_option("", "--ldxx", dest="ldxx",
help="Path to the cxx linker to use. (Xcode Distinction)",
type=str, default=None)
group.add_option("", "--test-externals", dest="test_suite_externals",
help="Path to the LLVM test-suite externals",
type=str, default=None, metavar="PATH")
group.add_option("", "--machine-param", dest="machine_parameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the machine parameters",
type=str, action="append", default=[])
group.add_option("", "--run-param", dest="run_parameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the run parameters",
type=str, action="append", default=[])
group.add_option("", "--run-order", dest="run_order", metavar="STR",
help="String to use to identify and order this run",
action="store", type=str, default=None)
group.add_option("", "--test-subdir", dest="test_subdir",
help="Subdirectory of test external dir to look for tests in.",
type=str, default="lnt-compile-suite-src")
parser.add_option_group(group)
group = OptionGroup(parser, "Test Selection")
group.add_option("", "--no-memory-profiling", dest="memory_profiling",
help="Disable memory profiling",
action="store_false", default=True)
group.add_option("", "--multisample", dest="run_count", metavar="N",
help="Accumulate test data from multiple runs",
action="store", type=int, default=3)
group.add_option("", "--min-sample-time", dest="min_sample_time",
help="Ensure all tests run for at least N seconds",
metavar="N", action="store", type=float, default=.5)
group.add_option("", "--save-temps", dest="save_temps",
help="Save temporary build output files",
action="store_true", default=False)
group.add_option("", "--show-tests", dest="show_tests",
help="Only list the availables tests that will be run",
action="store_true", default=False)
group.add_option("", "--test", dest="tests", metavar="NAME",
help="Individual test to run",
action="append", default=[])
group.add_option("", "--test-filter", dest="test_filters",
help="Run tests matching the given pattern",
metavar="REGEXP", action="append", default=[])
group.add_option("", "--flags-to-test", dest="flags_to_test",
help="Add a set of flags to test (space separated)",
metavar="FLAGLIST", action="append", default=[])
group.add_option("", "--jobs-to-test", dest="jobs_to_test",
help="Add a job count to test (full builds)",
metavar="NUM", action="append", default=[], type=int)
group.add_option("", "--config-to-test", dest="configs_to_test",
help="Add build configuration to test (full builds)",
metavar="NAME", action="append", default=[],
choices=('Debug', 'Release'))
parser.add_option_group(group)
group = OptionGroup(parser, "Output Options")
group.add_option("", "--no-machdep-info", dest="use_machdep_info",
help=("Don't put machine (instance) dependent "
"variables in machine info"),
action="store_false", default=True)
group.add_option("", "--machine-name", dest="machine_name", type='str',
help="Machine name to use in submission [%default]",
action="store", default=platform.uname()[1])
group.add_option("", "--submit", dest="submit_url", metavar="URLORPATH",
help=("autosubmit the test result to the given server "
"(or local instance) [%default]"),
type=str, default=None)
group.add_option("", "--commit", dest="commit",
help=("whether the autosubmit result should be committed "
"[%default]"),
type=int, default=True)
group.add_option("", "--output", dest="output", metavar="PATH",
help="write raw report data to PATH (or stdout if '-')",
action="store", default=None)
group.add_option("-v", "--verbose", dest="verbose",
help="show verbose test results",
action="store_true", default=False)
parser.add_option_group(group)
opts, args = parser.parse_args(args)
if len(args) != 0:
parser.error("invalid number of arguments")
if opts.cc is None:
parser.error("You must specify a --cc argument.")
# Resolve the cc_under_test path.
opts.cc = resolve_command_path(opts.cc)
if not lnt.testing.util.compilers.is_valid(opts.cc):
parser.error('--cc does not point to a valid executable.')
# Attempt to infer the cxx compiler if not given.
if opts.cc and opts.cxx is None:
opts.cxx = lnt.testing.util.compilers.infer_cxx_compiler(opts.cc)
if opts.cxx is not None:
note("inferred C++ compiler under test as: %r" % (opts.cxx,))
# Validate options.
if opts.cc is None:
parser.error('--cc is required')
if opts.cxx is None:
parser.error('--cxx is required (and could not be inferred)')
if opts.sandbox_path is None:
parser.error('--sandbox is required')
if opts.test_suite_externals is None:
parser.error("--test-externals option is required")
# Force the CC and CXX variables to be absolute paths.
cc_abs = os.path.abspath(commands.which(opts.cc))
cxx_abs = os.path.abspath(commands.which(opts.cxx))
if not os.path.exists(cc_abs):
parser.error("unable to determine absolute path for --cc: %r" % (
opts.cc,))
if not os.path.exists(cxx_abs):
parser.error("unable to determine absolute path for --cc: %r" % (
opts.cc,))
opts.cc = cc_abs
opts.cxx = cxx_abs
# If no ld was set, set ld to opts.cc
if opts.ld is None:
opts.ld = opts.cc
# If no ldxx was set, set ldxx to opts.cxx
if opts.ldxx is None:
opts.ldxx = opts.cxx
# Set up the sandbox.
global g_output_dir
if not os.path.exists(opts.sandbox_path):
print >>sys.stderr, "%s: creating sandbox: %r" % (
timestamp(), opts.sandbox_path)
os.mkdir(opts.sandbox_path)
if opts.timestamp_build:
fmt_timestamp = timestamp().replace(' ', '_').replace(':', '-')
report_name = "test-%s" % (fmt_timestamp)
else:
report_name = "build"
g_output_dir = os.path.join(os.path.abspath(opts.sandbox_path),
report_name)
try:
os.mkdir(g_output_dir)
except OSError(e):
if e.errno == errno.EEXIST:
parser.error("sandbox output directory %r already exists!" % (
g_output_dir,))
else:
raise
# Setup log file
global g_log
def setup_log(output_dir):
def stderr_log_handler():
h = logging.StreamHandler()
f = logging.Formatter("%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
def file_log_handler(path):
h = logging.FileHandler(path, mode='w')
f = logging.Formatter("%(asctime)-7s: %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
h.setFormatter(f)
return h
l = logging.Logger('compile_test')
l.setLevel(logging.INFO)
l.addHandler(file_log_handler(os.path.join(output_dir, 'test.log')))
l.addHandler(stderr_log_handler())
return l
g_log = setup_log(g_output_dir)
# Collect machine and run information.
machine_info, run_info = machineinfo.get_machine_information(
opts.use_machdep_info)
# FIXME: Include information on test source versions.
#
# FIXME: Get more machine information? Cocoa.h hash, for example.
for name, cmd in (('sys_cc_version', ('/usr/bin/gcc', '-v')),
('sys_as_version', ('/usr/bin/as', '-v', '/dev/null')),
('sys_ld_version', ('/usr/bin/ld', '-v')),
('sys_xcodebuild', ('xcodebuild', '-version'))):
run_info[name] = commands.capture(cmd, include_stderr=True).strip()
# Set command line machine and run information.
for info, params in ((machine_info, opts.machine_parameters),
(run_info, opts.run_parameters)):
for entry in params:
if '=' not in entry:
name, value = entry, ''
else:
name, value = entry.split('=', 1)
info[name] = value
# Set user variables.
variables = {}
variables['cc'] = opts.cc
variables['run_count'] = opts.run_count
# Get compiler info.
cc_info = lnt.testing.util.compilers.get_cc_info(variables['cc'])
variables.update(cc_info)
# Set the run order from the user, if given.
if opts.run_order is not None:
variables['run_order'] = opts.run_order
else:
# Otherwise, use the inferred run order.
variables['run_order'] = cc_info['inferred_run_order']
note("inferred run order to be: %r" % (variables['run_order'],))
if opts.verbose:
format = pprint.pformat(variables)
msg = '\n\t'.join(['using variables:'] + format.splitlines())
note(msg)
format = pprint.pformat(machine_info)
msg = '\n\t'.join(['using machine info:'] + format.splitlines())
note(msg)
format = pprint.pformat(run_info)
msg = '\n\t'.join(['using run info:'] + format.splitlines())
note(msg)
# Compute the set of flags to test.
if not opts.flags_to_test:
flags_to_test = DEFAULT_FLAGS_TO_TEST
else:
flags_to_test = [string.split(' ')
for string in opts.flags_to_test]
# Compute the set of job counts to use in full build tests.
if not opts.jobs_to_test:
jobs_to_test = [1, 2, 4, 8]
else:
jobs_to_test = opts.jobs_to_test
# Compute the build configurations to test.
if not opts.configs_to_test:
configs_to_test = ['Debug', 'Release']
else:
configs_to_test = opts.configs_to_test
# Compute the list of all tests.
all_tests = list(get_tests(opts.test_suite_externals, opts.test_subdir,
flags_to_test, jobs_to_test,
configs_to_test))
# Show the tests, if requested.
if opts.show_tests:
print >>sys.stderr, 'Available Tests'
for name in sorted(set(name for name, _ in all_tests)):
print >>sys.stderr, ' %s' % (name,)
print
raise SystemExit
# Find the tests to run.
if not opts.tests and not opts.test_filters:
tests_to_run = list(all_tests)
else:
all_test_names = set(test[0] for test in all_tests)
# Validate the test names.
requested_tests = set(opts.tests)
missing_tests = requested_tests - all_test_names
if missing_tests:
parser.error(("invalid test names %s, use --show-tests to "
"see available tests") %
(", ".join(map(repr, missing_tests)), ))
# Validate the test filters.
test_filters = [re.compile(pattern)
for pattern in opts.test_filters]
# Form the list of tests.
tests_to_run = [test
for test in all_tests
if (test[0] in requested_tests or
[True
for filter in test_filters
if filter.search(test[0])])]
if not tests_to_run:
parser.error(
"no tests requested (invalid --test or --test-filter options)!")
# Ensure output directory is available.
if not os.path.exists(g_output_dir):
os.mkdir(g_output_dir)
# Execute the run.
run_info.update(variables)
run_info['tag'] = tag = 'compile'
testsamples = []
start_time = datetime.utcnow()
g_log.info('run started')
g_log.info('using CC: %r' % opts.cc)
g_log.info('using CXX: %r' % opts.cxx)
for basename, test_fn in tests_to_run:
for success, name, samples in test_fn(basename, run_info,
variables):
g_log.info('collected samples: %r' % name)
num_samples = len(samples)
if num_samples:
samples_median = '%.4f' % (stats.median(samples),)
samples_mad = '%.4f' % (
stats.median_absolute_deviation(samples),)
else:
samples_median = samples_mad = 'N/A'
g_log.info('N=%d, median=%s, MAD=%s' % (
num_samples, samples_median, samples_mad))
test_name = '%s.%s' % (tag, name)
if not success:
testsamples.append(lnt.testing.TestSamples(
test_name + '.status',
[lnt.testing.FAIL]))
if samples:
testsamples.append(lnt.testing.TestSamples(
test_name, samples))
end_time = datetime.utcnow()
g_log.info('run complete')
# Package up the report.
machine = lnt.testing.Machine(opts.machine_name, machine_info)
run = lnt.testing.Run(start_time, end_time, info=run_info)
# Write out the report.
lnt_report_path = os.path.join(g_output_dir, 'report.json')
report = lnt.testing.Report(machine, run, testsamples)
# Save report to disk for submission.
self.print_report(report, lnt_report_path)
# Then, also print to screen if requested.
if opts.output is not None:
self.print_report(report, opts.output)
server_report = self.submit(lnt_report_path, opts)
return server_report
