def playback_benchmark(options, args):
"""Using the target build configuration, run the playback test."""
root_dir = os.path.dirname(options.build_dir) # That's src dir.
data_dir = os.path.join(root_dir, 'data', 'webapp_benchmarks', 'gmailjs')
benchmark_results = {'ready': threading.Event()}
def callback(results):
benchmark_results['results'] = results
benchmark_results['ready'].set()
benchmark = playback_benchmark_replay.ReplayBenchmark(
callback, data_dir, SERVER_PORT)
server_thread = threading.Thread(target=benchmark.RunForever)
server_thread.setDaemon(True)
server_thread.start()
if chromium_utils.IsLinux():
xvfb.StartVirtualX(options.target, '')
result = run_benchmark(options, False, benchmark_results)
result |= run_benchmark(options, True, benchmark_results)
if chromium_utils.IsLinux():
xvfb.StopVirtualX(options.target)
return result
def RunPythonCommandInBuildDir(build_dir,
target,
command_line_args,
server_dir=None):
if sys.platform == 'win32':
python_exe = 'python.exe'
setup_mount = chromium_utils.FindUpward(build_dir, 'third_party',
'cygwin', 'setup_mount.bat')
chromium_utils.RunCommand([setup_mount])
else:
os.environ['PYTHONPATH'] = (
chromium_utils.FindUpward(build_dir, 'tools', 'python') + ":" +
os.environ.get('PYTHONPATH', ''))
python_exe = 'python'
if chromium_utils.IsLinux():
slave_name = SlaveBuildName(build_dir)
xvfb.StartVirtualX(slave_name,
os.path.join(build_dir, '..', 'out', target),
server_dir=server_dir)
command = [python_exe]
# The list of tests is given as arguments.
command.extend(command_line_args)
result = chromium_utils.RunCommand(command)
if chromium_utils.IsLinux():
xvfb.StopVirtualX(slave_name)
return result
def _Main(options, args, extra_env):
"""Using the target build configuration, run the executable given in the
first non-option argument, passing any following arguments to that
executable.
Args:
options: Command-line options for this invocation of runtest.py.
args: Command and arguments for the test.
extra_env: A dictionary of extra environment variables to set.
Returns:
Exit status code.
"""
if len(args) < 1:
raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
xvfb_path = os.path.join(os.path.dirname(sys.argv[0]), '..', '..',
'third_party', 'xvfb',
platform.architecture()[0])
build_dir = os.path.normpath(os.path.abspath(options.build_dir))
bin_dir = os.path.join(build_dir, options.target)
test_exe = args[0]
if options.run_python_script:
test_exe_path = test_exe
else:
test_exe_path = os.path.join(bin_dir, test_exe)
if not os.path.exists(test_exe_path):
if options.factory_properties.get('succeed_on_missing_exe', False):
print '%s missing but succeed_on_missing_exe used, exiting' % (
test_exe_path)
return 0
raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path)
if sys.platform == 'linux2':
# Unset http_proxy and HTTPS_PROXY environment variables. When set, this
# causes some tests to hang. See http://crbug.com/139638 for more info.
if 'http_proxy' in os.environ:
del os.environ['http_proxy']
print 'Deleted http_proxy environment variable.'
if 'HTTPS_PROXY' in os.environ:
del os.environ['HTTPS_PROXY']
print 'Deleted HTTPS_PROXY environment variable.'
# Path to SUID sandbox binary. This must be installed on all bots.
extra_env['CHROME_DEVEL_SANDBOX'] = CHROME_SANDBOX_PATH
extra_env['LD_LIBRARY_PATH'] = ''
if options.enable_lsan:
# Use the debug version of libstdc++ under LSan. If we don't, there will
# be a lot of incomplete stack traces in the reports.
extra_env['LD_LIBRARY_PATH'] += '/usr/lib/x86_64-linux-gnu/debug:'
extra_env['LD_LIBRARY_PATH'] += '%s:%s/lib:%s/lib.target' % (
bin_dir, bin_dir, bin_dir)
if options.run_shell_script:
command = ['bash', test_exe_path]
elif options.run_python_script:
command = [sys.executable, test_exe]
else:
command = _BuildTestBinaryCommand(build_dir, test_exe_path, options)
command.extend(args[1:])
log_processor = None
try:
# TODO(dpranke): checking on test_exe is a temporary hack until we
# can change the buildbot master to pass --xvfb instead of --no-xvfb
# for these two steps. See
# https://code.google.com/p/chromium/issues/detail?id=179814
start_xvfb = (sys.platform == 'linux2'
and (options.xvfb or 'layout_test_wrapper' in test_exe
or 'devtools_perf_test_wrapper' in test_exe))
if start_xvfb:
xvfb.StartVirtualX(None,
bin_dir,
with_wm=(options.factory_properties.get(
'window_manager', 'True') == 'True'))
if options.test_launcher_summary_output:
command.append('--test-launcher-summary-output=%s' %
options.test_launcher_summary_output)
command = _GenerateRunIsolatedCommand(build_dir, test_exe_path,
options, command)
env = os.environ.copy()
if extra_env:
print 'Additional test environment:'
for k, v in sorted(extra_env.items()):
print ' %s=%s' % (k, v)
env.update(extra_env or {})
# Trigger bot mode (test retries, redirection of stdio, possibly faster,
# etc.) - using an environment variable instead of command-line flags
# because some internal waterfalls run this for totally non-gtest code.
# TODO(phajdan.jr): Clean this up when internal waterfalls are fixed.
env.update({'CHROMIUM_TEST_LAUNCHER_BOT_MODE': '1'})
if options.use_symbolization_script:
symbolize_command = _GetSanitizerSymbolizeCommand(
strip_path_prefix=options.strip_path_prefix)
command_process = subprocess.Popen(command,
env=env,
stdout=subprocess.PIPE)
symbolize_process = subprocess.Popen(symbolize_command,
env=env,
stdin=command_process.stdout)
command_process.stdout.close()
command_process.wait()
symbolize_process.wait()
result = command_process.returncode
if result == 0:
result = symbolize_process.returncode
else:
result = subprocess.call(command, env=env)
finally:
if start_xvfb:
xvfb.StopVirtualX(None)
return result
def dom_perf(options, args):
"""Using the target build configuration, run the dom perf test."""
build_dir = os.path.abspath(options.build_dir)
if chromium_utils.IsWindows():
test_exe_name = 'performance_ui_tests.exe'
else:
test_exe_name = 'performance_ui_tests'
if chromium_utils.IsMac():
is_make_or_ninja = (options.factory_properties.get(
"gclient_env", {}).get('GYP_GENERATORS', '') in ('ninja', 'make'))
if is_make_or_ninja:
build_dir = os.path.join(os.path.dirname(build_dir), 'out')
else:
build_dir = os.path.join(os.path.dirname(build_dir), 'xcodebuild')
elif chromium_utils.IsLinux():
build_dir = os.path.join(os.path.dirname(build_dir), 'sconsbuild')
test_exe_path = os.path.join(build_dir, options.target, test_exe_name)
if not os.path.exists(test_exe_path):
raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path)
# Find the current revision to pass to the test.
build_revision = slave_utils.SubversionRevision(build_dir)
# Compute the path to the test data.
src_dir = os.path.dirname(build_dir)
data_dir = os.path.join(src_dir, 'data')
dom_perf_dir = os.path.join(data_dir, 'dom_perf')
iterations = '' # Default
if options.target == 'Debug':
iterations = '&minIterations=1'
def run_and_print(use_refbuild):
# Windows used to write to the root of C:, but that doesn't work
# on Vista so we write into the build folder instead.
suffix = ''
if (use_refbuild):
suffix = '_ref'
output_file = os.path.join(
build_dir, options.target,
'dom_perf_result_%s%s.txt' % (build_revision, suffix))
result = 0
compiled_data = []
for test in TESTS:
url = URL % (dom_perf_dir, test, iterations, build_revision)
url_flag = '--url=%s' % url
command = [
test_exe_path, '--wait_cookie_name=__domperf_finished',
'--jsvar=__domperf_result',
'--jsvar_output=%s' % output_file,
'--gtest_filter=UrlFetchTest.UrlFetch', url_flag
]
if use_refbuild:
command.append('--reference_build')
print "Executing: "
print command
result |= chromium_utils.RunCommand(command)
# Open the resulting file and display it.
data = json.load(open(output_file, 'r'))
for suite in data['BenchmarkSuites']:
# Skip benchmarks that we didn't actually run this time around.
if len(suite['Benchmarks']) == 0 and suite['score'] == 0:
continue
compiled_data.append(suite)
# Now give the geometric mean as the total for the combined runs.
total = geometric_mean([s['score'] for s in compiled_data])
print_result(True, 'Total', total, use_refbuild)
for suite in compiled_data:
print_result(False, suite['name'], suite['score'], use_refbuild)
return result
try:
if chromium_utils.IsLinux():
xvfb.StartVirtualX(options.target,
os.path.join(build_dir, options.target))
result = run_and_print(False)
result |= run_and_print(True)
finally:
if chromium_utils.IsLinux():
xvfb.StopVirtualX(options.target)
return result
def main_linux(options, args):
if len(args) < 1:
raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
build_dir = os.path.normpath(os.path.abspath(options.build_dir))
slave_name = slave_utils.SlaveBuildName(build_dir)
# If this is a sub-project build (i.e. there's a 'sconsbuild' in build_dir),
# look for the test binaries there, otherwise look for the top-level build
# output.
# This assumes we never pass a build_dir which might contain build output that
# we're not trying to test. This is currently a safe assumption since we don't
# have any builders that do both sub-project and top-level builds (only
# Modules builders do sub-project builds), so they shouldn't ever have both
# 'build_dir/sconsbuild' and 'build_dir/../sconsbuild'.
outdir = None
if os.path.exists(os.path.join(build_dir, 'sconsbuild')):
outdir = 'sconsbuild'
elif os.path.exists(os.path.join(build_dir, 'out')):
outdir = 'out'
if outdir:
bin_dir = os.path.join(build_dir, outdir, options.target)
src_dir = os.path.join(slave_utils.SlaveBaseDir(build_dir), 'build',
'src')
os.environ['CR_SOURCE_ROOT'] = src_dir
else:
if os.path.exists(os.path.join(build_dir, '..', 'sconsbuild')):
bin_dir = os.path.join(build_dir, '..', 'sconsbuild',
options.target)
else:
bin_dir = os.path.join(build_dir, '..', 'out', options.target)
# Figure out what we want for a special frame buffer directory.
special_xvfb_dir = None
if options.special_xvfb == 'auto':
fp_special_xvfb = options.factory_properties.get('special_xvfb', None)
fp_chromeos = options.factory_properties.get('chromeos', None)
if fp_special_xvfb or (
fp_special_xvfb is None and
(fp_chromeos or slave_utils.GypFlagIsOn(options, 'use_aura')
or slave_utils.GypFlagIsOn(options, 'chromeos'))):
special_xvfb_dir = options.special_xvfb_dir
elif options.special_xvfb:
special_xvfb_dir = options.special_xvfb_dir
test_exe = args[0]
test_exe_path = os.path.join(bin_dir, test_exe)
if not os.path.exists(test_exe_path):
if options.factory_properties.get('succeed_on_missing_exe', False):
print '%s missing but succeed_on_missing_exe used, exiting' % (
test_exe_path)
return 0
msg = 'Unable to find %s' % test_exe_path
raise chromium_utils.PathNotFound(msg)
# Decide whether to enable the suid sandbox for Chrome.
if should_enable_sandbox(CHROME_SANDBOX_PATH):
print 'Enabling sandbox. Setting environment variable:'
print ' CHROME_DEVEL_SANDBOX="%s"' % CHROME_SANDBOX_PATH
os.environ['CHROME_DEVEL_SANDBOX'] = CHROME_SANDBOX_PATH
else:
print 'Disabling sandbox. Setting environment variable:'
print ' CHROME_DEVEL_SANDBOX=""'
os.environ['CHROME_DEVEL_SANDBOX'] = ''
# Nuke anything that appears to be stale chrome items in the temporary
# directory from previous test runs (i.e.- from crashes or unittest leaks).
slave_utils.RemoveChromeTemporaryFiles()
os.environ['LD_LIBRARY_PATH'] = '%s:%s/lib:%s/lib.target' % (
bin_dir, bin_dir, bin_dir)
# Figure out what we want for a special llvmpipe directory.
if (options.llvmpipe_dir and os.path.exists(options.llvmpipe_dir)):
os.environ['LD_LIBRARY_PATH'] += ':' + options.llvmpipe_dir
if options.parallel:
command = _BuildParallelCommand(build_dir, test_exe_path, options)
elif options.run_shell_script:
command = ['bash', test_exe_path]
elif options.run_python_script:
command = [sys.executable, test_exe]
else:
command = [test_exe_path]
command.extend(args[1:])
results_tracker = None
if options.generate_json_file:
results_tracker = gtest_slave_utils.GTestUnexpectedDeathTracker()
if os.path.exists(options.test_output_xml):
# remove the old XML output file.
os.remove(options.test_output_xml)
try:
http_server = None
if options.document_root:
http_server = start_http_server(
'linux',
build_dir=build_dir,
test_exe_path=test_exe_path,
document_root=options.document_root)
if options.xvfb:
xvfb.StartVirtualX(slave_name,
bin_dir,
with_wm=options.factory_properties.get(
'window_manager', True),
server_dir=special_xvfb_dir)
if options.factory_properties.get('asan', False):
symbolize = os.path.abspath(
os.path.join('src', 'tools', 'valgrind', 'asan',
'asan_symbolize.py'))
pipes = [[sys.executable, symbolize], ['c++filt']]
result = _RunGTestCommand(command, pipes=pipes)
else:
result = _RunGTestCommand(command, results_tracker)
finally:
if http_server:
http_server.StopServer()
if options.xvfb:
xvfb.StopVirtualX(slave_name)
if options.generate_json_file:
_GenerateJSONForTestResults(options, results_tracker)
return result
def _Main(options, args, extra_env):
"""Using the target build configuration, run the executable given in the
first non-option argument, passing any following arguments to that
executable.
Args:
options: Command-line options for this invocation of runtest.py.
args: Command and arguments for the test.
extra_env: A dictionary of extra environment variables to set.
Returns:
Exit status code.
"""
if len(args) < 1:
raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
xvfb_path = os.path.join(os.path.dirname(sys.argv[0]), '..', '..',
'third_party', 'xvfb',
platform.architecture()[0])
special_xvfb_dir = None
fp_chromeos = options.factory_properties.get('chromeos', None)
if (fp_chromeos or slave_utils.GypFlagIsOn(options, 'use_aura')
or slave_utils.GypFlagIsOn(options, 'chromeos')):
special_xvfb_dir = xvfb_path
build_dir = os.path.normpath(os.path.abspath(options.build_dir))
bin_dir = os.path.join(build_dir, options.target)
slave_name = options.slave_name or slave_utils.SlaveBuildName(build_dir)
test_exe = args[0]
if options.run_python_script:
test_exe_path = test_exe
else:
test_exe_path = os.path.join(bin_dir, test_exe)
if not os.path.exists(test_exe_path):
if options.factory_properties.get('succeed_on_missing_exe', False):
print '%s missing but succeed_on_missing_exe used, exiting' % (
test_exe_path)
return 0
raise chromium_utils.PathNotFound('Unable to find %s' % test_exe_path)
if sys.platform == 'linux2':
# Unset http_proxy and HTTPS_PROXY environment variables. When set, this
# causes some tests to hang. See http://crbug.com/139638 for more info.
if 'http_proxy' in os.environ:
del os.environ['http_proxy']
print 'Deleted http_proxy environment variable.'
if 'HTTPS_PROXY' in os.environ:
del os.environ['HTTPS_PROXY']
print 'Deleted HTTPS_PROXY environment variable.'
# Path to SUID sandbox binary. This must be installed on all bots.
extra_env['CHROME_DEVEL_SANDBOX'] = CHROME_SANDBOX_PATH
extra_env['LD_LIBRARY_PATH'] = ''
if options.enable_lsan:
# Use the debug version of libstdc++ under LSan. If we don't, there will
# be a lot of incomplete stack traces in the reports.
extra_env['LD_LIBRARY_PATH'] += '/usr/lib/x86_64-linux-gnu/debug:'
extra_env['LD_LIBRARY_PATH'] += '%s:%s/lib:%s/lib.target' % (
bin_dir, bin_dir, bin_dir)
if options.run_shell_script:
command = ['bash', test_exe_path]
elif options.run_python_script:
command = [sys.executable, test_exe]
else:
command = _BuildTestBinaryCommand(build_dir, test_exe_path, options)
command.extend(args[1:])
# Nuke anything that appears to be stale chrome items in the temporary
# directory from previous test runs (i.e.- from crashes or unittest leaks).
slave_utils.RemoveChromeTemporaryFiles()
log_processor = None
if _UsingGtestJson(options):
log_processor = gtest_utils.GTestJSONParser(
options.build_properties.get('mastername'))
if options.generate_json_file:
if os.path.exists(options.test_output_xml):
# remove the old XML output file.
os.remove(options.test_output_xml)
try:
# TODO(dpranke): checking on test_exe is a temporary hack until we
# can change the buildbot master to pass --xvfb instead of --no-xvfb
# for these two steps. See
# https://code.google.com/p/chromium/issues/detail?id=179814
start_xvfb = (sys.platform == 'linux2'
and (options.xvfb or 'layout_test_wrapper' in test_exe
or 'devtools_perf_test_wrapper' in test_exe))
if start_xvfb:
xvfb.StartVirtualX(slave_name,
bin_dir,
with_wm=(options.factory_properties.get(
'window_manager', 'True') == 'True'),
server_dir=special_xvfb_dir)
if _UsingGtestJson(options):
json_file_name = log_processor.PrepareJSONFile(
options.test_launcher_summary_output)
command.append('--test-launcher-summary-output=%s' %
json_file_name)
pipes = []
# See the comment in main() regarding offline symbolization.
if options.use_symbolization_script:
symbolize_command = _GetSanitizerSymbolizeCommand(
strip_path_prefix=options.strip_path_prefix)
pipes = [symbolize_command]
command = _GenerateRunIsolatedCommand(build_dir, test_exe_path,
options, command)
result = _RunGTestCommand(options, command, extra_env, pipes=pipes)
finally:
if start_xvfb:
xvfb.StopVirtualX(slave_name)
if _UsingGtestJson(options):
if options.use_symbolization_script:
_SymbolizeSnippetsInJSON(options, json_file_name)
log_processor.ProcessJSONFile(options.build_dir)
if options.generate_json_file:
if not _GenerateJSONForTestResults(options, log_processor):
return 1
if options.annotate:
annotation_utils.annotate(options.test_type, result, log_processor)
return result