def run(self, cmake_vars, compile=True, test=True):
mkdir_p(self._base_path)
if self.opts.pgo:
self._collect_pgo(self._base_path)
self.trained = True
self.configured = False
self._configure_if_needed()
if self.compiled and compile:
self._clean(self._base_path)
if not self.compiled or compile:
self._make(self._base_path)
self.compiled = True
data = self._lit(self._base_path, test)
return self._parse_lit_output(self._base_path, data, cmake_vars), data
def run(self, nick, compile=True, test=True):
path = self._base_path
if not os.path.exists(path):
mkdir_p(path)
if not self.configured and self._need_to_configure(path):
self._configure(path)
self._clean(path)
self.configured = True
if self.compiled and compile:
self._clean(path)
if not self.compiled or compile:
self._make(path)
self.compiled = True
data = self._lit(path, test)
return self._parse_lit_output(path, data)
def run(self, nick, compile=True, test=True):
path = self._base_path
if not os.path.exists(path):
mkdir_p(path)
if not self.configured and self._need_to_configure(path):
self._configure(path)
self._clean(path)
self.configured = True
if self.compiled and compile:
self._clean(path)
if not self.compiled or compile:
self._make(path)
self.compiled = True
data = self._lit(path, test)
return self._parse_lit_output(path, data)
def run(self, cmake_vars, compile=True, test=True, profile=False):
mkdir_p(self._base_path)
if self.opts.pgo:
self._collect_pgo(self._base_path)
self.trained = True
self.configured = False
self._configure_if_needed()
if self.compiled and compile:
self._clean(self._base_path)
if not self.compiled or compile:
self._make(self._base_path)
self._install_benchmark(self._base_path)
self.compiled = True
data = self._lit(self._base_path, test, profile)
return self._parse_lit_output(self._base_path, data, cmake_vars), data
def run(self, cmake_vars, compile=True, test=True, profile=False):
mkdir_p(self._base_path)
# FIXME: should we only run PGO collection once, even when
# multisampling? We could do so be adding "and not self.trained"
# below.
if self.opts.pgo:
self._collect_pgo(self._base_path)
self.trained = True
self.configured = False
self._configure_if_needed()
if self.compiled and compile:
self._clean(self._base_path)
if not self.compiled or compile or self.opts.pgo:
self._make(self._base_path)
self._install_benchmark(self._base_path)
self.compiled = True
data = self._lit(self._base_path, test, profile)
return self._parse_lit_output(self._base_path, data, cmake_vars), data
def test_build(base_name,
run_info,
variables,
project,
build_config,
num_jobs,
codesize_util=None):
name = '%s(config=%r,j=%d)' % (base_name, build_config, num_jobs)
# Check if we need to expand the archive into the sandbox.
archive_path = get_input_path(opts, project['archive'])
with open(archive_path) as f:
archive_hash = hashlib.md5(f.read() + str(project)).hexdigest()
# Compute the path to unpack to.
source_path = get_output_path("..", "Sources", project['name'])
# Load the hash of the last unpack, in case the archive has been updated.
last_unpack_hash_path = os.path.join(source_path, "last_unpack_hash.txt")
if os.path.exists(last_unpack_hash_path):
with open(last_unpack_hash_path) as f:
last_unpack_hash = f.read()
else:
last_unpack_hash = None
# Unpack if necessary.
if last_unpack_hash == archive_hash:
g_log.info('reusing sources %r (already unpacked)' % name)
else:
# Remove any existing content, if necessary.
try:
shutil.rmtree(source_path)
except OSError, e:
if e.errno != errno.ENOENT:
raise
# Extract the zip file.
#
# We shell out to unzip here because zipfile's extractall does not
# appear to preserve permissions properly.
commands.mkdir_p(source_path)
g_log.info('extracting sources for %r' % name)
if archive_path[-6:] == "tar.gz":
p = subprocess.Popen(args=['tar', '-xzf', archive_path],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=source_path)
else:
p = subprocess.Popen(args=['unzip', '-q', archive_path],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=source_path)
stdout, stderr = p.communicate()
if p.wait() != 0:
fatal(("unable to extract archive %r at %r\n"
"-- stdout --\n%s\n"
"-- stderr --\n%s\n") %
(archive_path, source_path, stdout, stderr))
if p.wait() != 0:
fatal
# Apply the patch file, if necessary.
patch_files = project.get('patch_files', [])
for patch_file in patch_files:
g_log.info('applying patch file %r for %r' % (patch_file, name))
patch_file_path = get_input_path(opts, patch_file)
p = subprocess.Popen(
args=['patch', '-i', patch_file_path, '-p', '1'],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=source_path)
stdout, stderr = p.communicate()
if p.wait() != 0:
fatal(("unable to apply patch file %r in %r\n"
"-- stdout --\n%s\n"
"-- stderr --\n%s\n") %
(patch_file_path, source_path, stdout, stderr))
# Write the hash tag.
with open(last_unpack_hash_path, "w") as f:
f.write(archive_hash)
with open(last_unpack_hash_path, "w") as f:
f.write(archive_hash)
# Create an env dict in case the user wants to use it.
env = dict(os.environ)
# Form the test build command.
build_info = project['build_info']
# Add arguments to ensure output files go into our build directory.
dir_name = '%s_%s_j%d' % (base_name, build_config, num_jobs)
output_base = get_output_path(dir_name)
build_base = os.path.join(output_base, 'build', build_config)
# Create the build base directory and by extension output base directory.
commands.mkdir_p(build_base)
cmd = []
preprocess_cmd = None
if build_info['style'].startswith('xcode-'):
file_path = os.path.join(source_path, build_info['file'])
cmd.extend(['xcodebuild'])
# Add the arguments to select the build target.
if build_info['style'] == 'xcode-project':
cmd.extend(
('-target', build_info['target'], '-project', file_path))
elif build_info['style'] == 'xcode-workspace':
cmd.extend(
('-scheme', build_info['scheme'], '-workspace', file_path))
def diagnose(self):
"""Build a triage report that contains information about a test.
This is an alternate top level target for running the test-suite. It
will produce a triage report for a benchmark instead of running the
test-suite normally. The report has stuff in it that will be useful
for reproducing and diagnosing a performance change.
"""
assert self.opts.only_test, "We don't have a benchmark to diagenose."
bm_path, short_name = self.opts.only_test
assert bm_path, "The benchmark path is empty?"
report_name = "{}.report".format(short_name)
# Make a place for the report.
report_path = os.path.abspath(report_name)
# Overwrite the report.
if os.path.exists(report_path):
shutil.rmtree(report_path)
os.mkdir(report_path)
path = self._base_path
if not os.path.exists(path):
mkdir_p(path)
os.chdir(path)
# Run with -save-temps
cmd = self._configure(path, execute=False)
cmd_temps = cmd + [
'-DTEST_SUITE_DIAGNOSE=On',
'-DTEST_SUITE_DIAGNOSE_FLAGS=-save-temps'
]
note(' '.join(cmd_temps))
out = subprocess.check_output(cmd_temps)
note(out)
# Figure out our test's target.
make_cmd = [self.opts.make, "VERBOSE=1", 'help']
make_targets = subprocess.check_output(make_cmd)
matcher = re.compile(r"^\.\.\.\s{}$".format(short_name),
re.MULTILINE | re.IGNORECASE)
if not matcher.search(make_targets):
assert False, "did not find benchmark, must be nestsed? Unimplemented."
local_path = os.path.join(path, bm_path)
make_save_temps = [self.opts.make, "VERBOSE=1", short_name]
note(" ".join(make_save_temps))
out = subprocess.check_output(make_save_temps)
note(out)
# Executable(s) and test file:
shutil.copy(os.path.join(local_path, short_name), report_path)
shutil.copy(os.path.join(local_path, short_name + ".test"),
report_path)
# Temp files are in:
temp_files = os.path.join(local_path, "CMakeFiles",
short_name + ".dir")
save_temps_file = ["/*.s", "/*.ii", "/*.i", "/*.bc"]
build_files = [
"/*.o", "/*.time", "/*.cmake", "/*.make", "/*.includecache",
"/*.txt"
]
self._cp_artifacts(local_path, report_path, save_temps_file)
self._cp_artifacts(temp_files, report_path, build_files)
note("Report produced in: " + report_path)
# Run through the rest of LNT, but don't allow this to be submitted
# because there is no data.
class DontSubmitResults(object):
def get(self, url):
return None
return DontSubmitResults()
def test_build(base_name, run_info, variables, project, build_config, num_jobs,
codesize_util=None):
name = '%s(config=%r,j=%d)' % (base_name, build_config, num_jobs)
# Check if we need to expand the archive into the sandbox.
archive_path = get_input_path(opts, project['archive'])
with open(archive_path) as f:
archive_hash = hashlib.md5(f.read() + str(project)).hexdigest()
# Compute the path to unpack to.
source_path = get_output_path("..", "Sources", project['name'])
# Load the hash of the last unpack, in case the archive has been updated.
last_unpack_hash_path = os.path.join(source_path, "last_unpack_hash.txt")
if os.path.exists(last_unpack_hash_path):
with open(last_unpack_hash_path) as f:
last_unpack_hash = f.read()
else:
last_unpack_hash = None
# Unpack if necessary.
if last_unpack_hash == archive_hash:
g_log.info('reusing sources %r (already unpacked)' % name)
else:
# Remove any existing content, if necessary.
try:
shutil.rmtree(source_path)
except OSError, e:
if e.errno != errno.ENOENT:
raise
# Extract the zip file.
#
# We shell out to unzip here because zipfile's extractall does not
# appear to preserve permissions properly.
commands.mkdir_p(source_path)
g_log.info('extracting sources for %r' % name)
if archive_path[-6:] == "tar.gz":
p = subprocess.Popen(args=['tar', '-xzf', archive_path],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=source_path)
else:
p = subprocess.Popen(args=['unzip', '-q', archive_path],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=source_path)
stdout,stderr = p.communicate()
if p.wait() != 0:
fatal(("unable to extract archive %r at %r\n"
"-- stdout --\n%s\n"
"-- stderr --\n%s\n") % (archive_path, source_path,
stdout, stderr))
if p.wait() != 0:
fatal
# Apply the patch file, if necessary.
patch_files = project.get('patch_files', [])
for patch_file in patch_files:
g_log.info('applying patch file %r for %r' % (patch_file, name))
patch_file_path = get_input_path(opts, patch_file)
p = subprocess.Popen(args=['patch', '-i', patch_file_path,
'-p', '1'],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=source_path)
stdout,stderr = p.communicate()
if p.wait() != 0:
fatal(("unable to apply patch file %r in %r\n"
"-- stdout --\n%s\n"
"-- stderr --\n%s\n") % (patch_file_path, source_path,
stdout, stderr))
# Write the hash tag.
with open(last_unpack_hash_path, "w") as f:
f.write(archive_hash)
with open(last_unpack_hash_path, "w") as f:
f.write(archive_hash)
# Create an env dict in case the user wants to use it.
env = dict(os.environ)
# Form the test build command.
build_info = project['build_info']
# Add arguments to ensure output files go into our build directory.
dir_name = '%s_%s_j%d' % (base_name, build_config, num_jobs)
output_base = get_output_path(dir_name)
build_base = os.path.join(output_base, 'build', build_config)
# Create the build base directory and by extension output base directory.
commands.mkdir_p(build_base)
cmd = []
preprocess_cmd = None
if build_info['style'].startswith('xcode-'):
file_path = os.path.join(source_path, build_info['file'])
cmd.extend(['xcodebuild'])
# Add the arguments to select the build target.
if build_info['style'] == 'xcode-project':
cmd.extend(('-target', build_info['target'],
'-project', file_path))
elif build_info['style'] == 'xcode-workspace':
cmd.extend(('-scheme', build_info['scheme'],
'-workspace', file_path))
def diagnose(self):
"""Build a triage report that contains information about a test.
This is an alternate top level target for running the test-suite. It
will produce a triage report for a benchmark instead of running the
test-suite normally. The report has stuff in it that will be useful
for reproducing and diagnosing a performance change.
"""
assert self.opts.only_test, "We don't have a benchmark to diagenose."
bm_path, short_name = self.opts.only_test
assert bm_path, "The benchmark path is empty?"
report_name = "{}.report".format(short_name)
# Make a place for the report.
report_path = os.path.abspath(report_name)
# Overwrite the report.
if os.path.exists(report_path):
shutil.rmtree(report_path)
os.mkdir(report_path)
path = self._base_path
if not os.path.exists(path):
mkdir_p(path)
os.chdir(path)
# Run with -save-temps
cmd = self._configure(path, execute=False)
cmd_temps = cmd + ['-DTEST_SUITE_DIAGNOSE=On',
'-DTEST_SUITE_DIAGNOSE_FLAGS=-save-temps']
note(' '.join(cmd_temps))
out = subprocess.check_output(cmd_temps)
note(out)
# Figure out our test's target.
make_cmd = [self.opts.make, "VERBOSE=1", 'help']
make_targets = subprocess.check_output(make_cmd)
matcher = re.compile(r"^\.\.\.\s{}$".format(short_name),
re.MULTILINE | re.IGNORECASE)
if not matcher.search(make_targets):
assert False, "did not find benchmark, must be nestsed? Unimplemented."
local_path = os.path.join(path, bm_path)
make_save_temps = [self.opts.make, "VERBOSE=1", short_name]
note(" ".join(make_save_temps))
out = subprocess.check_output(make_save_temps)
note(out)
# Executable(s) and test file:
shutil.copy(os.path.join(local_path, short_name), report_path)
shutil.copy(os.path.join(local_path, short_name + ".test"), report_path)
# Temp files are in:
temp_files = os.path.join(local_path, "CMakeFiles",
short_name + ".dir")
save_temps_file = ["/*.s", "/*.ii", "/*.i", "/*.bc"]
build_files = ["/*.o", "/*.time", "/*.cmake", "/*.make",
"/*.includecache", "/*.txt"]
self._cp_artifacts(local_path, report_path, save_temps_file)
self._cp_artifacts(temp_files, report_path, build_files)
note("Report produced in: " + report_path)
# Run through the rest of LNT, but don't allow this to be submitted
# because there is no data.
class DontSubmitResults(object):
def get(self, url):
return None
return DontSubmitResults()
def diagnose(self):
"""Build a triage report that contains information about a test.
This is an alternate top level target for running the test-suite. It
will produce a triage report for a benchmark instead of running the
test-suite normally. The report has stuff in it that will be useful
for reproducing and diagnosing a performance change.
"""
assert self.opts.only_test, "We don't have a benchmark to diagenose."
bm_path, short_name = self.opts.only_test
assert bm_path, "The benchmark path is empty?"
report_name = "{}.report".format(short_name)
# Make a place for the report.
report_path = os.path.abspath(report_name)
# Overwrite the report.
if os.path.exists(report_path):
shutil.rmtree(report_path)
os.mkdir(report_path)
path = self._base_path
mkdir_p(path)
os.chdir(path)
# Run with -save-temps
cmd = self._configure(path, execute=False)
cmd_temps = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-save-temps']
note(' '.join(cmd_temps))
out = subprocess.check_output(cmd_temps)
note(out)
# Figure out our test's target.
make_cmd = [self.opts.make, "VERBOSE=1", 'help']
make_targets = subprocess.check_output(make_cmd)
matcher = re.compile(r"^\.\.\.\s{}$".format(short_name),
re.MULTILINE | re.IGNORECASE)
if not matcher.search(make_targets):
assert False, "did not find benchmark, nestsed? Unimplemented."
local_path = os.path.join(path, bm_path)
make_deps = [self.opts.make, "VERBOSE=1", "timeit-target",
"timeit-host", "fpcmp-host"]
note(" ".join(make_deps))
p = subprocess.Popen(make_deps,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
std_out, std_err = p.communicate()
note(std_out)
make_save_temps = [self.opts.make, "VERBOSE=1", short_name]
note(" ".join(make_save_temps))
p = subprocess.Popen(make_save_temps,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
std_out, std_err = p.communicate()
note(std_out)
with open(report_path + "/build.log", 'w') as f:
f.write(std_out)
# Executable(s) and test file:
shutil.copy(os.path.join(local_path, short_name), report_path)
shutil.copy(os.path.join(local_path, short_name + ".test"), report_path)
# Temp files are in:
temp_files = os.path.join(local_path, "CMakeFiles",
short_name + ".dir")
save_temps_file = ["/*.s", "/*.ii", "/*.i", "/*.bc"]
build_files = ["/*.o", "/*.time", "/*.cmake", "/*.make",
"/*.includecache", "/*.txt"]
self._cp_artifacts(local_path, report_path, save_temps_file)
self._cp_artifacts(temp_files, report_path, build_files)
# Now lets do -ftime-report.
cmd_time_report = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-ftime-report']
note(' '.join(cmd_time_report))
out = subprocess.check_output(cmd_time_report)
note(out)
make_time_report = [self.opts.make, "VERBOSE=1", short_name]
p = subprocess.Popen(make_time_report,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
with open(report_path + "/time-report.txt", 'w') as f:
f.write(std_err)
note("Wrote: " + report_path + "/time-report.txt")
# Now lets do -llvm -stats.
cmd_stats_report = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-mllvm -stats']
note(' '.join(cmd_stats_report))
out = subprocess.check_output(cmd_stats_report)
note(out)
make_stats_report = [self.opts.make, "VERBOSE=1", short_name]
p = subprocess.Popen(make_stats_report,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
with open(report_path + "/stats-report.txt", 'w') as f:
f.write(std_err)
note("Wrote: " + report_path + "/stats-report.txt")
# Collect Profile:
if "Darwin" in platform.platform():
# For testing and power users, lets allow overrides of how sudo
# and iprofiler are called.
sudo = os.getenv("SUDO_CMD", "sudo")
if " " in sudo:
sudo = sudo.split(" ")
if not sudo:
sudo = []
else:
sudo = [sudo]
iprofiler = os.getenv("IPROFILER_CMD",
"iprofiler -timeprofiler -I 40u")
cmd_iprofiler = cmd + ['-DTEST_SUITE_RUN_UNDER=' + iprofiler]
print ' '.join(cmd_iprofiler)
out = subprocess.check_output(cmd_iprofiler)
os.chdir(local_path)
make_iprofiler_temps = [self.opts.make, "VERBOSE=1", short_name]
p = subprocess.Popen(make_iprofiler_temps,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
warning("Using sudo to collect execution trace.")
make_save_temps = sudo + [self.opts.lit, short_name + ".test"]
p = subprocess.Popen(make_save_temps,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
sys.stdout.write(std_out)
sys.stderr.write(std_err)
warning("Tests may fail because of iprofiler's output.")
# The dtps file will be saved as root, make it so
# that we can read it.
chmod = sudo + ["chown", "-R", getpass.getuser(),
short_name + ".dtps"]
subprocess.call(chmod)
profile = local_path + "/" + short_name + ".dtps"
shutil.copytree(profile, report_path + "/" + short_name + ".dtps")
note(profile + "-->" + report_path)
else:
warning("Skipping execution profiling because this is not Darwin.")
note("Report produced in: " + report_path)
# Run through the rest of LNT, but don't allow this to be submitted
# because there is no data.
class DontSubmitResults(object):
def get(self, url):
return report_path
def __getitem__(self, key):
return report_path
return DontSubmitResults()
def _configure_if_needed(self):
mkdir_p(self._base_path)
if not self.configured:
self._configure(self._base_path)
self._clean(self._base_path)
self.configured = True
def test_build(base_name,
run_info,
variables,
project,
build_config,
num_jobs,
codesize_util=None):
name = '%s(config=%r,j=%d)' % (base_name, build_config, num_jobs)
# Check if we need to expand the archive into the sandbox.
archive_path = get_input_path(opts, project['archive'])
with open(archive_path) as f:
archive_hash = hashlib.md5(f.read() + str(project)).hexdigest()
# Compute the path to unpack to.
source_path = get_output_path("..", "Sources", project['name'])
# Load the hash of the last unpack, in case the archive has been updated.
last_unpack_hash_path = os.path.join(source_path, "last_unpack_hash.txt")
if os.path.exists(last_unpack_hash_path):
with open(last_unpack_hash_path) as f:
last_unpack_hash = f.read()
else:
last_unpack_hash = None
# Unpack if necessary.
if last_unpack_hash == archive_hash:
g_log.info('reusing sources %r (already unpacked)' % name)
else:
# Remove any existing content, if necessary.
try:
shutil.rmtree(source_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
# Extract the zip file.
#
# We shell out to unzip here because zipfile's extractall does not
# appear to preserve permissions properly.
commands.mkdir_p(source_path)
g_log.info('extracting sources for %r' % name)
if archive_path.endswith(".tar.gz") or \
archive_path.endswith(".tar.bz2") or \
archive_path.endswith(".tar.lzma"):
p = subprocess.Popen(args=['tar', '-xf', archive_path],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=source_path,
universal_newlines=True)
else:
p = subprocess.Popen(args=['unzip', '-q', archive_path],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=source_path,
universal_newlines=True)
stdout, stderr = p.communicate()
if p.wait() != 0:
fatal(("unable to extract archive %r at %r\n"
"-- stdout --\n%s\n"
"-- stderr --\n%s\n") %
(archive_path, source_path, stdout, stderr))
# Apply the patch file, if necessary.
patch_files = project.get('patch_files', [])
for patch_file in patch_files:
g_log.info('applying patch file %r for %r' % (patch_file, name))
patch_file_path = get_input_path(opts, patch_file)
p = subprocess.Popen(
args=['patch', '-i', patch_file_path, '-p', '1'],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=source_path,
universal_newlines=True)
stdout, stderr = p.communicate()
if p.wait() != 0:
fatal(("unable to apply patch file %r in %r\n"
"-- stdout --\n%s\n"
"-- stderr --\n%s\n") %
(patch_file_path, source_path, stdout, stderr))
# Write the hash tag.
with open(last_unpack_hash_path, "w") as f:
f.write(archive_hash)
# Create an env dict in case the user wants to use it.
env = dict(os.environ)
# Form the test build command.
build_info = project['build_info']
# Add arguments to ensure output files go into our build directory.
dir_name = '%s_%s_j%d' % (base_name, build_config, num_jobs)
output_base = get_output_path(dir_name)
build_base = os.path.join(output_base, 'build', build_config)
# Create the build base directory and by extension output base directory.
commands.mkdir_p(build_base)
cmd = []
preprocess_cmd = None
if build_info['style'].startswith('xcode-'):
file_path = os.path.join(source_path, build_info['file'])
cmd.extend(['xcodebuild'])
# Add the arguments to select the build target.
if build_info['style'] == 'xcode-project':
cmd.extend(
('-target', build_info['target'], '-project', file_path))
elif build_info['style'] == 'xcode-workspace':
cmd.extend(
('-scheme', build_info['scheme'], '-workspace', file_path))
cmd.extend(('-derivedDataPath', build_base))
else:
fatal("unknown build style in project: %r" % project)
# Add the build configuration selection.
cmd.extend(('-configuration', build_config))
cmd.append('OBJROOT=%s' % os.path.join(build_base, 'obj'))
cmd.append('SYMROOT=%s' % os.path.join(build_base, 'sym'))
cmd.append('DSTROOT=%s' % os.path.join(build_base, 'dst'))
cmd.append('SHARED_PRECOMPS_DIR=%s' % os.path.join(build_base, 'pch'))
# Add arguments to force the appropriate compiler.
cmd.append('CC=%s' % (opts.cc, ))
cmd.append('CPLUSPLUS=%s' % (opts.cxx, ))
# We need to force this variable here because Xcode has some completely
# broken logic for deriving this variable from the compiler
# name. <rdar://problem/7989147>
cmd.append('LD=%s' % (opts.ld, ))
cmd.append('LDPLUSPLUS=%s' % (opts.ldxx, ))
# Force off the static analyzer, in case it was enabled in any projects
# (we don't want to obscure what we are trying to time).
cmd.append('RUN_CLANG_STATIC_ANALYZER=NO')
# Inhibit all warnings, we don't want to count the time to generate
# them against newer compilers which have added (presumably good)
# warnings.
cmd.append('GCC_WARN_INHIBIT_ALL_WARNINGS=YES')
# Add additional arguments to force the build scenario we want.
cmd.extend(('-jobs', str(num_jobs)))
# If the user specifies any additional options to be included on the
# command line, append them here.
cmd.extend(build_info.get('extra_args', []))
# If the user specifies any extra environment variables, put
# them in our env dictionary.
env_format = {'build_base': build_base}
extra_env = build_info.get('extra_env', {})
for k in extra_env:
extra_env[k] = extra_env[k] % env_format
env.update(extra_env)
# Create preprocess cmd
preprocess_cmd = 'rm -rf "%s"' % (build_base, )
elif build_info['style'] == 'make':
# Get the subdirectory in Source where our sources exist.
src_dir = os.path.dirname(os.path.join(source_path,
build_info['file']))
# Grab our config from build_info. This is config is currently only
# used in the make build style since Xcode, the only other build style
# as of today, handles changing configuration through the configuration
# type variables. Make does not do this so we have to use more brute
# force to get it right.
config = build_info.get('config', {}).get(build_config, {})
# Copy our source directory over to build_base.
# We do this since we assume that we are processing a make project
# which has already been configured and so that we do not need to worry
# about make install or anything like that. We can just build the
# project and use the user supplied path to its location in the build
# directory.
copied_src_dir = os.path.join(build_base, os.path.basename(dir_name))
shutil.copytree(src_dir, copied_src_dir)
# Create our make command.
cmd.extend([
'make', '-C', copied_src_dir, build_info['target'], "-j",
str(num_jobs)
])
# If the user specifies any additional options to be included on the
# command line, append them here.
cmd.extend(config.get('extra_args', []))
# If the user specifies any extra environment variables, put
# them in our env dictionary.
# We create a dictionary for build_base so that users can use
# it optionally in an environment variable via the python
# format %(build_base)s.
env_format = {'build_base': build_base}
extra_env = config.get('extra_env', {})
for k in extra_env:
extra_env[k] = extra_env[k] % env_format
env.update(extra_env)
# Set build base to copied_src_dir so that if codesize_util
# is not None, we pass it the correct path.
build_base = copied_src_dir
preprocess_cmd = 'rm -rf "%s"/build' % (build_base, )
g_log.info('preprocess_cmd: %s' % preprocess_cmd)
else:
fatal("unknown build style in project: %r" % project)
# Collect the samples.
g_log.info('executing full build: %s' % args_to_quoted_string(cmd))
stdout_path = os.path.join(output_base, "stdout.log")
stderr_path = os.path.join(output_base, "stderr.log")
for res in get_runN_test_data(name,
variables,
cmd,
stdout=stdout_path,
stderr=stderr_path,
preprocess_cmd=preprocess_cmd,
env=env):
yield res
# If we have a binary path, get the text size of our result.
binary_path = build_info.get('binary_path', None)
if binary_path is not None and codesize_util is not None:
tname = "%s.size" % (name, )
success = False
samples = []
try:
# We use a dictionary here for our formatted processing of
# binary_path so that if the user needs our build config he can get
# it via %(build_config)s in his string and if he does not, an
# error is not thrown.
format_args = {"build_config": build_config}
cmd = codesize_util + [
os.path.join(build_base, binary_path % format_args)
]
if opts.verbose:
g_log.info('running: %s' % " ".join("'%s'" % arg
for arg in cmd))
result = subprocess.check_output(cmd).strip()
if result != "fail":
bytes = int(result)
success = True
# For now, the way the software is set up things are going to
# get confused if we don't report the same number of samples
# as reported for other variables. So we just report the size
# N times.
#
# FIXME: We should resolve this, eventually.
for i in range(variables.get('run_count')):
samples.append(bytes)
else:
g_log.warning('Codesize failed.')
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
g_log.warning('Codesize failed with ENOENT.')
yield (success, tname, samples)
# Check that the file sizes of the output log files "make sense", and warn
# if they do not. That might indicate some kind of non-determinism in the
# test command, which makes timing less useful.
stdout_sizes = []
stderr_sizes = []
run_count = variables['run_count']
for i in range(run_count):
iter_stdout_path = '%s.%d' % (stdout_path, i)
iter_stderr_path = '%s.%d' % (stderr_path, i)
if os.path.exists(iter_stdout_path):
stdout_sizes.append(os.stat(iter_stdout_path).st_size)
else:
stdout_sizes.append(None)
if os.path.exists(iter_stderr_path):
stderr_sizes.append(os.stat(iter_stderr_path).st_size)
else:
stderr_sizes.append(None)
if len(set(stdout_sizes)) != 1:
g_log.warning(('test command had stdout files with '
'different sizes: %r') % stdout_sizes)
if len(set(stderr_sizes)) != 1:
g_log.warning(('test command had stderr files with '
'different sizes: %r') % stderr_sizes)
# Unless cleanup is disabled, rerun the preprocessing command.
if not opts.save_temps and preprocess_cmd:
g_log.info('cleaning up temporary results')
if os.system(preprocess_cmd) != 0:
g_log.warning("cleanup command returned a non-zero exit status")
def diagnose(self):
"""Build a triage report that contains information about a test.
This is an alternate top level target for running the test-suite. It
will produce a triage report for a benchmark instead of running the
test-suite normally. The report has stuff in it that will be useful
for reproducing and diagnosing a performance change.
"""
assert self.opts.only_test, "We don't have a benchmark to diagnose."
bm_path, short_name = self.opts.only_test
assert bm_path, "The benchmark path is empty?"
report_name = "{}.report".format(short_name)
# Make a place for the report.
report_path = os.path.abspath(report_name)
# Overwrite the report.
if os.path.exists(report_path):
shutil.rmtree(report_path)
os.mkdir(report_path)
path = self._base_path
mkdir_p(path)
os.chdir(path)
# Run with -save-temps
cmd = self._configure(path, execute=False)
cmd_temps = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-save-temps']
logger.info(' '.join(cmd_temps))
out = subprocess.check_output(cmd_temps)
logger.info(out)
# Figure out our test's target.
make_cmd = [self.opts.make, "VERBOSE=1", 'help']
make_targets = subprocess.check_output(make_cmd)
matcher = re.compile(r"^\.\.\.\s{}$".format(short_name),
re.MULTILINE | re.IGNORECASE)
if not matcher.search(make_targets):
assert False, "did not find benchmark, nestsed? Unimplemented."
local_path = os.path.join(path, bm_path)
make_deps = [self.opts.make, "VERBOSE=1", "timeit-target",
"timeit-host", "fpcmp-host"]
logger.info(" ".join(make_deps))
p = subprocess.Popen(make_deps,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
std_out, std_err = p.communicate()
logger.info(std_out)
make_save_temps = [self.opts.make, "VERBOSE=1", short_name]
logger.info(" ".join(make_save_temps))
p = subprocess.Popen(make_save_temps,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
std_out, std_err = p.communicate()
logger.info(std_out)
with open(report_path + "/build.log", 'w') as f:
f.write(std_out)
# Executable(s) and test file:
shutil.copy(os.path.join(local_path, short_name), report_path)
shutil.copy(os.path.join(local_path, short_name + ".test"),
report_path)
# Temp files are in:
temp_files = os.path.join(local_path, "CMakeFiles",
short_name + ".dir")
save_temps_file = ["/*.s", "/*.ii", "/*.i", "/*.bc"]
build_files = ["/*.o", "/*.time", "/*.cmake", "/*.make",
"/*.includecache", "/*.txt"]
self._cp_artifacts(local_path, report_path, save_temps_file)
self._cp_artifacts(temp_files, report_path, build_files)
# Now lets do -ftime-report.
cmd_time_report = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-ftime-report']
logger.info(' '.join(cmd_time_report))
out = subprocess.check_output(cmd_time_report)
logger.info(out)
make_time_report = [self.opts.make, "VERBOSE=1", short_name]
p = subprocess.Popen(make_time_report,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
with open(report_path + "/time-report.txt", 'w') as f:
f.write(std_err)
logger.info("Wrote: " + report_path + "/time-report.txt")
# Now lets do -llvm -stats.
cmd_stats_report = cmd + ['-DTEST_SUITE_DIAGNOSE_FLAGS=-mllvm -stats']
logger.info(' '.join(cmd_stats_report))
out = subprocess.check_output(cmd_stats_report)
logger.info(out)
make_stats_report = [self.opts.make, "VERBOSE=1", short_name]
p = subprocess.Popen(make_stats_report,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
with open(report_path + "/stats-report.txt", 'w') as f:
f.write(std_err)
logger.info("Wrote: " + report_path + "/stats-report.txt")
# Collect Profile:
if "Darwin" in platform.platform():
# For testing and power users, lets allow overrides of how sudo
# and iprofiler are called.
sudo = os.getenv("SUDO_CMD", "sudo")
if " " in sudo:
sudo = sudo.split(" ")
if not sudo:
sudo = []
else:
sudo = [sudo]
iprofiler = os.getenv("IPROFILER_CMD",
"iprofiler -timeprofiler -I 40u")
cmd_iprofiler = cmd + ['-DTEST_SUITE_RUN_UNDER=' + iprofiler]
print ' '.join(cmd_iprofiler)
out = subprocess.check_output(cmd_iprofiler)
os.chdir(local_path)
make_iprofiler_temps = [self.opts.make, "VERBOSE=1", short_name]
p = subprocess.Popen(make_iprofiler_temps,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
logger.warning("Using sudo to collect execution trace.")
make_save_temps = sudo + [self.opts.lit, short_name + ".test"]
p = subprocess.Popen(make_save_temps,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
sys.stdout.write(std_out)
sys.stderr.write(std_err)
logger.warning("Tests may fail because of iprofiler's output.")
# The dtps file will be saved as root, make it so
# that we can read it.
chmod = sudo + ["chown", "-R", getpass.getuser(),
short_name + ".dtps"]
subprocess.call(chmod)
profile = local_path + "/" + short_name + ".dtps"
shutil.copytree(profile, report_path + "/" + short_name + ".dtps")
logger.info(profile + "-->" + report_path)
else:
logger.warning("Skipping execution profiling because " +
"this is not Darwin.")
logger.info("Report produced in: " + report_path)
return lnt.util.ImportData.no_submit()
def _configure_if_needed(self):
mkdir_p(self._base_path)
if not self.configured:
self._configure(self._base_path)
self._clean(self._base_path)
self.configured = True
