def RunTestsSingly(exe, tests):
for test in tests:
filter = test
if len(sys.argv) > 2:
filter = filter + ":" + sys.argv[2]
cmd = [exe, "--gtest_filter=" + filter]
common.RunSubprocess(cmd)
def SimpleTest(self, module, name, valgrind_test_args=None, cmd_args=None):
cmd = self._DefaultCommand(module, name, valgrind_test_args)
self._ReadGtestFilterFile(name, cmd)
if cmd_args:
cmd.extend(["--"])
cmd.extend(cmd_args)
return common.RunSubprocess(cmd, 0)
def Execute(self):
""" Execute the app to be tested after successful instrumentation.
Full execution command-line provided by subclassers via proc."""
logging.info("starting execution...")
proc = self.ToolCommand()
for var in self._env:
common.PutEnvAndLog(var, self._env[var])
return common.RunSubprocess(proc, self._timeout)
def Execute(self):
"""Executes the app to be tested."""
logging.info('starting execution...')
proc = self._args
self.PutEnvAndLog('G_SLICE', 'always-malloc')
self.PutEnvAndLog('NSS_DISABLE_ARENA_FREE_LIST', '1')
self.PutEnvAndLog('NSS_DISABLE_UNLOAD', '1')
self.PutEnvAndLog('GTEST_DEATH_TEST_USE_FORK', '1')
return common.RunSubprocess(proc, self._timeout)
def InstrumentDll(self):
'''Does a blocking Purify instrumentation of chrome.dll.'''
# TODO(paulg): Make this code support any DLL.
cmd = self._DefaultCommand("chrome")
cmd.append("--instrument_only")
cmd.append(os.path.join(self._options.build_dir, "chrome.dll"))
result = common.RunSubprocess(cmd, 0)
if result:
logging.error("Instrumentation error: %d" % result)
return result
def SimpleTest(self, module, name):
cmd = self._DefaultCommand(module, name)
if not self._options.run_singly:
self._ReadGtestFilterFile(name, cmd)
cmd.append("--gtest_print_time")
return common.RunSubprocess(cmd, 0)
else:
exe = cmd[-1]
script = ["python.exe", "test_runner.py", exe]
return self.ScriptedTest(module, exe, name, script, multi=True)
def TestLayoutChunk(self, chunk_num, chunk_size):
# Run tests [chunk_num*chunk_size .. (chunk_num+1)*chunk_size) from the
# list of tests. Wrap around to beginning of list at end.
# If chunk_size is zero, run all tests in the list once.
# If a text file is given as argument, it is used as the list of tests.
#
# Build the ginormous commandline in 'cmd'.
# It's going to be roughly
# python valgrind_test.py ... python run_webkit_tests.py ...
# but we'll use the --indirect flag to valgrind_test.py
# to avoid valgrinding python.
# Start by building the valgrind_test.py commandline.
cmd = self._DefaultCommand("webkit")
cmd.append("--trace_children")
cmd.append("--indirect")
# Now build script_cmd, the run_webkits_tests.py commandline
# Store each chunk in its own directory so that we can find the data later
chunk_dir = os.path.join("layout", "chunk_%05d" % chunk_num)
test_shell = os.path.join(self._options.build_dir, "test_shell")
out_dir = os.path.join(google.path_utils.ScriptDir(), "latest")
out_dir = os.path.join(out_dir, chunk_dir)
if os.path.exists(out_dir):
old_files = glob.glob(os.path.join(out_dir, "*.txt"))
for f in old_files:
os.remove(f)
else:
os.makedirs(out_dir)
script = os.path.join(self._source_dir, "webkit", "tools", "layout_tests",
"run_webkit_tests.py")
script_cmd = ["python", script, "--run-singly", "-v",
"--noshow-results", "--time-out-ms=200000",
"--nocheck-sys-deps"]
# Pass build mode to run_webkit_tests.py. We aren't passed it directly,
# so parse it out of build_dir. run_webkit_tests.py can only handle
# the two values "Release" and "Debug".
# TODO(Hercules): unify how all our scripts pass around build mode
# (--mode / --target / --build_dir / --debug)
if self._options.build_dir.endswith("Debug"):
script_cmd.append("--debug");
if (chunk_size > 0):
script_cmd.append("--run-chunk=%d:%d" % (chunk_num, chunk_size))
if len(self._args):
# if the arg is a txt file, then treat it as a list of tests
if os.path.isfile(self._args[0]) and self._args[0][-4:] == ".txt":
script_cmd.append("--test-list=%s" % self._args[0])
else:
script_cmd.extend(self._args)
self._ReadGtestFilterFile("layout", script_cmd)
# Now run script_cmd with the wrapper in cmd
cmd.extend(["--"])
cmd.extend(script_cmd)
ret = common.RunSubprocess(cmd, 0)
return ret
def Execute(self):
''' Execute the app to be tested after successful instrumentation.
Full execution command-line provided by subclassers via proc.'''
logging.info("starting execution...")
proc = self.ValgrindCommand()
os.putenv("G_SLICE", "always-malloc")
logging.info("export G_SLICE=always-malloc");
os.putenv("NSS_DISABLE_ARENA_FREE_LIST", "1")
logging.info("export NSS_DISABLE_ARENA_FREE_LIST=1");
os.putenv("GTEST_DEATH_TEST_USE_FORK", "1")
logging.info("export GTEST_DEATH_TEST_USE_FORK=1");
return common.RunSubprocess(proc, self._timeout)
def ScriptedTest(self,
module,
exe,
name,
script,
multi=False,
cmd_args=None,
out_dir_extra=None):
'''Purify a target exe, which will be executed one or more times via a
script or driver program.
Args:
module - which top level component this test is from (webkit, base, etc.)
exe - the name of the exe (it's assumed to exist in build_dir)
name - the name of this test (used to name output files)
script - the driver program or script. If it's python.exe, we use
search-path behavior to execute, otherwise we assume that it is in
build_dir.
multi - a boolean hint that the exe will be run multiple times, generating
multiple output files (without this option, only the last run will be
recorded and analyzed)
cmd_args - extra arguments to pass to the purify_test.py script
'''
if out_dir_extra:
self._report_dir = os.path.join(self._report_dir, out_dir_extra)
cmd = self._DefaultCommand(module)
exe = os.path.join(self._options.build_dir, exe)
cmd.append("--exe=%s" % exe)
cmd.append("--name=%s" % name)
if multi:
if out_dir_extra:
if os.path.exists(self._report_dir):
old_files = glob.glob(
os.path.join(self._report_dir, "*.txt"))
for f in old_files:
os.remove(f)
else:
os.makedirs(self._report_dir)
out_file = os.path.join(self._report_dir, "%s%%5d.txt" % name)
cmd.append("--out_file=%s" % out_file)
if cmd_args:
cmd.extend(cmd_args)
if script[0] != "python.exe" and not os.path.exists(script[0]):
script[0] = os.path.join(self._options.build_dir, script[0])
cmd.extend(script)
self._ReadGtestFilterFile(name, cmd)
return common.RunSubprocess(cmd, 0)
def Execute(self):
"""Executes the app to be tested."""
logging.info('starting execution...')
proc = ['sh', path_utils.ScriptDir() + '/heapcheck_std.sh']
proc += self._args
self.PutEnvAndLog('G_SLICE', 'always-malloc')
self.PutEnvAndLog('NSS_DISABLE_ARENA_FREE_LIST', '1')
self.PutEnvAndLog('NSS_DISABLE_UNLOAD', '1')
self.PutEnvAndLog('GTEST_DEATH_TEST_USE_FORK', '1')
self.PutEnvAndLog('HEAPCHECK', self._mode)
self.PutEnvAndLog('HEAP_CHECK_ERROR_EXIT_CODE', '0')
self.PutEnvAndLog('HEAP_CHECK_MAX_LEAKS', '-1')
self.PutEnvAndLog('KEEP_SHADOW_STACKS', '1')
self.PutEnvAndLog(
'PPROF_PATH',
path_utils.ScriptDir() +
'/../../third_party/tcmalloc/chromium/src/pprof')
self.PutEnvAndLog('LD_LIBRARY_PATH',
'/usr/lib/debug/:/usr/lib32/debug/')
return common.RunSubprocess(proc, self._timeout)
def Cleanup(self):
common.Rational.Cleanup(self)
if self._instrument_only:
return
cmd = self._PurifyCommand()
# undo the /Replace=yes that was done in Instrument(), which means to
# remove the instrumented exe, and then rename exe.Original back to exe.
cmd.append("/UndoReplace")
cmd.append(os.path.abspath(self._exe))
common.RunSubprocess(cmd, self._timeout, detach=True)
# if we overwrote an existing ini file, restore it
ini_file = self._exe.replace(".exe", "_pure.ini")
if os.path.isfile(ini_file):
os.remove(ini_file)
ini_file_orig = ini_file + ".Original"
if os.path.isfile(ini_file_orig):
os.rename(ini_file_orig, ini_file)
# remove the pft file we wrote out
pft_file = self._exe.replace(".exe", "_exe.pft")
if os.path.isfile(pft_file):
os.remove(pft_file)
def Execute(self):
"""Executes the app to be tested."""
logging.info('starting execution...')
proc = ['sh', google.path_utils.ScriptDir() + '/heapcheck_std.sh']
proc += self._args
self.PutEnvAndLog('G_SLICE', 'always-malloc')
self.PutEnvAndLog('NSS_DISABLE_ARENA_FREE_LIST', '1')
self.PutEnvAndLog('GTEST_DEATH_TEST_USE_FORK', '1')
self.PutEnvAndLog('HEAPCHECK', self._mode)
self.PutEnvAndLog('HEAP_CHECK_MAX_LEAKS', '-1')
self.PutEnvAndLog(
'PPROF_PATH',
google.path_utils.ScriptDir() +
'/../../third_party/tcmalloc/chromium/src/pprof')
common.RunSubprocess(proc, self._timeout)
# Always return true, even if running the subprocess failed. We depend on
# Analyze to determine if the run was valid. (This behaviour copied from
# the purify_test.py script.)
return True
def Execute(self):
"""Executes the app to be tested."""
logging.info('starting execution...')
proc = ['sh', path_utils.ScriptDir() + '/heapcheck_std.sh']
proc += self._args
self.PutEnvAndLog('G_SLICE', 'always-malloc')
self.PutEnvAndLog('NSS_DISABLE_ARENA_FREE_LIST', '1')
self.PutEnvAndLog('NSS_DISABLE_UNLOAD', '1')
self.PutEnvAndLog('GTEST_DEATH_TEST_USE_FORK', '1')
self.PutEnvAndLog('HEAPCHECK', self._mode)
self.PutEnvAndLog('HEAP_CHECK_MAX_LEAKS', '-1')
self.PutEnvAndLog('KEEP_SHADOW_STACKS', '1')
self.PutEnvAndLog(
'PPROF_PATH',
path_utils.ScriptDir() +
'/../../third_party/tcmalloc/chromium/src/pprof')
self.PutEnvAndLog('LD_LIBRARY_PATH',
'/usr/lib/debug/:/usr/lib32/debug/')
common.RunSubprocess(proc, self._timeout)
# Always return true, even if running the subprocess failed. We depend on
# Analyze to determine if the run was valid.
return True
def ToolCommand(self):
"""Get the tool command to run."""
# WINHEAP is what Dr. Memory supports as there are issues w/ both
# jemalloc (http://code.google.com/p/drmemory/issues/detail?id=320) and
# tcmalloc (http://code.google.com/p/drmemory/issues/detail?id=314)
add_env = {
"CHROME_ALLOCATOR" : "WINHEAP",
"JSIMD_FORCEMMX" : "1", # http://code.google.com/p/drmemory/issues/detail?id=540
}
for k,v in add_env.iteritems():
logging.info("export %s=%s", k, v)
os.putenv(k, v)
drmem_cmd = os.getenv("DRMEMORY_COMMAND")
if not drmem_cmd:
raise RuntimeError, "Please set DRMEMORY_COMMAND environment variable " \
"with the path to drmemory.exe"
proc = drmem_cmd.split(" ")
# By default, don't run python (this will exclude python's children as well)
# to reduce runtime. We're not really interested in spending time finding
# bugs in the python implementation.
# With file-based config we must update the file every time, and
# it will affect simultaneous drmem uses by this user. While file-based
# config has many advantages, here we may want this-instance-only
# (http://code.google.com/p/drmemory/issues/detail?id=334).
drconfig_cmd = [ proc[0].replace("drmemory.exe", "drconfig.exe") ]
drconfig_cmd += ["-quiet"] # suppress errors about no 64-bit libs
run_drconfig = True
if self._options.follow_python:
logging.info("Following python children")
# -unreg fails if not already registered so query for that first
query_cmd = drconfig_cmd + ["-isreg", "python.exe"]
query_proc = subprocess.Popen(query_cmd, stdout=subprocess.PIPE,
shell=True)
(query_out, query_err) = query_proc.communicate()
if re.search("exe not registered", query_out):
run_drconfig = False # all set
else:
drconfig_cmd += ["-unreg", "python.exe"]
else:
logging.info("Excluding python children")
drconfig_cmd += ["-reg", "python.exe", "-norun"]
if run_drconfig:
drconfig_retcode = common.RunSubprocess(drconfig_cmd, self._timeout)
if drconfig_retcode:
logging.error("Configuring whether to follow python children failed " \
"with %d.", drconfig_retcode)
raise RuntimeError, "Configuring python children failed "
suppression_count = 0
supp_files = self._options.suppressions
if self.full_mode:
supp_files += [s.replace(".txt", "_full.txt") for s in supp_files]
for suppression_file in supp_files:
if os.path.exists(suppression_file):
suppression_count += 1
proc += ["-suppress", common.NormalizeWindowsPath(suppression_file)]
if not suppression_count:
logging.warning("WARNING: NOT USING SUPPRESSIONS!")
# Un-comment to dump Dr.Memory events on error
#proc += ["-dr_ops", "-dumpcore_mask", "-dr_ops", "0x8bff"]
# Un-comment and comment next line to debug Dr.Memory
#proc += ["-dr_ops", "-no_hide"]
#proc += ["-dr_ops", "-msgbox_mask", "-dr_ops", "15"]
#Proc += ["-dr_ops", "-stderr_mask", "-dr_ops", "15"]
# Ensure we see messages about Dr. Memory crashing!
proc += ["-dr_ops", "-stderr_mask", "-dr_ops", "12"]
if self._options.use_debug:
proc += ["-debug"]
proc += ["-logdir", common.NormalizeWindowsPath(self.log_dir)]
if self.log_parent_dir:
# gpu process on Windows Vista+ runs at Low Integrity and can only
# write to certain directories (http://crbug.com/119131)
symcache_dir = os.path.join(self.log_parent_dir, "drmemory.symcache")
elif self._options.build_dir:
# The other case is only possible with -t cmdline.
# Anyways, if we omit -symcache_dir the -logdir's value is used which
# should be fine.
symcache_dir = os.path.join(self._options.build_dir, "drmemory.symcache")
if symcache_dir:
if not os.path.exists(symcache_dir):
try:
os.mkdir(symcache_dir)
except OSError:
logging.warning("Can't create symcache dir?")
if os.path.exists(symcache_dir):
proc += ["-symcache_dir", common.NormalizeWindowsPath(symcache_dir)]
# Use -no_summary to suppress DrMemory's summary and init-time
# notifications. We generate our own with drmemory_analyze.py.
proc += ["-batch", "-no_summary"]
# Un-comment to disable interleaved output. Will also suppress error
# messages normally printed to stderr.
#proc += ["-quiet", "-no_results_to_stderr"]
proc += ["-callstack_max_frames", "40"]
# disable leak scan for now
proc += ["-no_count_leaks", "-no_leak_scan"]
# crbug.com/413215, no heap mismatch check for Windows release build binary
if common.IsWindows() and "Release" in self._options.build_dir:
proc += ["-no_check_delete_mismatch"]
# make callstacks easier to read
proc += ["-callstack_srcfile_prefix",
"build\\src,chromium\\src,crt_build\\self_x86"]
proc += ["-callstack_modname_hide",
"*drmemory*,chrome.dll"]
boring_callers = common.BoringCallers(mangled=False, use_re_wildcards=False)
# TODO(timurrrr): In fact, we want "starting from .." instead of "below .."
proc += ["-callstack_truncate_below", ",".join(boring_callers)]
if self.pattern_mode:
proc += ["-pattern", "0xf1fd", "-no_count_leaks", "-redzone_size", "0x20"]
elif not self.full_mode:
proc += ["-light"]
proc += self._tool_flags
# Dr.Memory requires -- to separate tool flags from the executable name.
proc += ["--"]
if self._options.indirect or self._options.indirect_webkit_layout:
# TODO(timurrrr): reuse for TSan on Windows
wrapper_path = os.path.join(self._source_dir,
"tools", "valgrind", "browser_wrapper_win.py")
wrapper = " ".join(["python", wrapper_path] + proc)
self.CreateBrowserWrapper(wrapper)
logging.info("browser wrapper = " + " ".join(proc))
if self._options.indirect_webkit_layout:
proc = self._args
# Layout tests want forward slashes.
wrapper = wrapper.replace('\\', '/')
proc += ["--wrapper", wrapper]
return proc
else:
proc = []
# Note that self._args begins with the name of the exe to be run.
self._args[0] = common.NormalizeWindowsPath(self._args[0])
proc += self._args
return proc
def PrepareForTestMac(self):
"""Runs dsymutil if needed.
Valgrind for Mac OS X requires that debugging information be in a .dSYM
bundle generated by dsymutil. It is not currently able to chase DWARF
data into .o files like gdb does, so executables without .dSYM bundles or
with the Chromium-specific "fake_dsym" bundles generated by
build/mac/strip_save_dsym won't give source file and line number
information in valgrind.
This function will run dsymutil if the .dSYM bundle is missing or if
it looks like a fake_dsym. A non-fake dsym that already exists is assumed
to be up-to-date.
"""
test_command = self._args[0]
dsym_bundle = self._args[0] + '.dSYM'
dsym_file = os.path.join(dsym_bundle, 'Contents', 'Resources', 'DWARF',
os.path.basename(test_command))
dsym_info_plist = os.path.join(dsym_bundle, 'Contents', 'Info.plist')
needs_dsymutil = True
saved_test_command = None
if os.path.exists(dsym_file) and os.path.exists(dsym_info_plist):
# Look for the special fake_dsym tag in dsym_info_plist.
dsym_info_plist_contents = open(dsym_info_plist).read()
if not re.search('^\s*<key>fake_dsym</key>$', dsym_info_plist_contents,
re.MULTILINE):
# fake_dsym is not set, this is a real .dSYM bundle produced by
# dsymutil. dsymutil does not need to be run again.
needs_dsymutil = False
else:
# fake_dsym is set. dsym_file is a copy of the original test_command
# before it was stripped. Copy it back to test_command so that
# dsymutil has unstripped input to work with. Move the stripped
# test_command out of the way, it will be restored when this is
# done.
saved_test_command = test_command + '.stripped'
os.rename(test_command, saved_test_command)
shutil.copyfile(dsym_file, test_command)
shutil.copymode(saved_test_command, test_command)
if needs_dsymutil:
if self._options.generate_dsym:
# Remove the .dSYM bundle if it exists.
shutil.rmtree(dsym_bundle, True)
dsymutil_command = ['dsymutil', test_command]
# dsymutil is crazy slow. Ideally we'd have a timeout here,
# but common.RunSubprocess' timeout is only checked
# after each line of output; dsymutil is silent
# until the end, and is then killed, which is silly.
common.RunSubprocess(dsymutil_command)
if saved_test_command:
os.rename(saved_test_command, test_command)
else:
logging.info("No real .dSYM for test_command. Line numbers will "
"not be shown. Either tell xcode to generate .dSYM "
"file, or use --generate_dsym option to this tool.")
def ToolCommand(self):
"""Get the tool command to run."""
tool_name = self.ToolName()
# WINHEAP is what Dr. Memory supports as there are issues w/ both
# jemalloc (http://code.google.com/p/drmemory/issues/detail?id=320) and
# tcmalloc (http://code.google.com/p/drmemory/issues/detail?id=314)
add_env = {
"CHROME_ALLOCATOR": "WINHEAP",
"JSIMD_FORCEMMX":
"1", # http://code.google.com/p/drmemory/issues/detail?id=540
}
for k, v in add_env.iteritems():
logging.info("export %s=%s", k, v)
os.putenv(k, v)
drmem_cmd = os.getenv("DRMEMORY_COMMAND")
if not drmem_cmd:
raise RuntimeError, "Please set DRMEMORY_COMMAND environment variable " \
"with the path to drmemory.exe"
proc = drmem_cmd.split(" ")
# By default, don't run python (this will exclude python's children as well)
# to reduce runtime. We're not really interested in spending time finding
# bugs in the python implementation.
# With file-based config we must update the file every time, and
# it will affect simultaneous drmem uses by this user. While file-based
# config has many advantages, here we may want this-instance-only
# (http://code.google.com/p/drmemory/issues/detail?id=334).
drconfig_cmd = [proc[0].replace("drmemory.exe", "drconfig.exe")]
drconfig_cmd += ["-quiet"] # suppress errors about no 64-bit libs
run_drconfig = True
if self._options.follow_python:
logging.info("Following python children")
# -unreg fails if not already registered so query for that first
query_cmd = drconfig_cmd + ["-isreg", "python.exe"]
query_proc = subprocess.Popen(query_cmd,
stdout=subprocess.PIPE,
shell=True)
(query_out, query_err) = query_proc.communicate()
if re.search("exe not registered", query_out):
run_drconfig = False # all set
else:
drconfig_cmd += ["-unreg", "python.exe"]
else:
logging.info("Excluding python children")
drconfig_cmd += ["-reg", "python.exe", "-norun"]
if run_drconfig:
drconfig_retcode = common.RunSubprocess(drconfig_cmd,
self._timeout)
if drconfig_retcode:
logging.error("Configuring whether to follow python children failed " \
"with %d.", drconfig_retcode)
raise RuntimeError, "Configuring python children failed "
suppression_count = 0
for suppression_file in self._options.suppressions:
if os.path.exists(suppression_file):
suppression_count += 1
proc += ["-suppress", suppression_file]
if not suppression_count:
logging.warning("WARNING: NOT USING SUPPRESSIONS!")
# Un-comment to dump Dr.Memory events on error
#proc += ["-dr_ops", "-dumpcore_mask 0x8bff"]
# Un-comment to debug Dr.Memory
#proc += ["-dr_ops", "-no_hide -msgbox_mask 15"]
if self._options.use_debug:
proc += ["-debug"]
proc += ["-logdir", self.log_dir]
proc += ["-batch", "-quiet", "-no_results_to_stderr"]
proc += ["-callstack_max_frames", "40"]
# make callstacks easier to read
proc += [
"-callstack_srcfile_prefix",
"build\\src,chromium\\src,crt_build\\self_x86"
]
proc += [
"-callstack_truncate_below", "main,BaseThreadInitThunk," +
"testing*Test*Run*,testing::internal::Handle*Exceptions*," +
"MessageLoop::Run," +
"RunnableMethod*,RunnableFunction*,DispatchToMethod*"
]
proc += ["-callstack_modname_hide", "*.exe,chrome.dll"]
if not self.handle_uninits_and_leaks:
proc += ["-no_check_uninitialized", "-no_count_leaks"]
proc += self._tool_flags
# Dr.Memory requires -- to separate tool flags from the executable name.
proc += ["--"]
if self._options.indirect:
self.CreateBrowserWrapper(" ".join(proc))
proc = []
# Note that self._args begins with the name of the exe to be run.
proc += self._args
return proc
