def ToolSpecificFlags(self):
ret = []
ignore_files = ["ignores.txt"]
for platform_suffix in common.PlatformNames():
ignore_files.append("ignores_%s.txt" % platform_suffix)
for ignore_file in ignore_files:
fullname = os.path.join(self._source_dir,
"tools", "valgrind", "tsan", ignore_file)
if os.path.exists(fullname):
fullname = common.NormalizeWindowsPath(fullname)
ret += ["--ignore=%s" % fullname]
# This should shorten filepaths for local builds.
ret += ["--file-prefix-to-cut=%s/" % self._source_dir]
# This should shorten filepaths on bots.
ret += ["--file-prefix-to-cut=build/src/"]
ret += ["--file-prefix-to-cut=out/Release/../../"]
# This should shorten filepaths for functions intercepted in TSan.
ret += ["--file-prefix-to-cut=scripts/tsan/tsan/"]
ret += ["--file-prefix-to-cut=src/tsan/tsan/"]
ret += ["--gen-suppressions=true"]
if self.EvalBoolFlag(self._options.hybrid):
ret += ["--hybrid=yes"] # "no" is the default value for TSAN
if self.EvalBoolFlag(self._options.announce_threads):
ret += ["--announce-threads"]
if self.EvalBoolFlag(self._options.free_is_write):
ret += ["--free-is-write=yes"]
else:
ret += ["--free-is-write=no"]
# --show-pc flag is needed for parsing the error logs on Darwin.
if platform_suffix == 'mac':
ret += ["--show-pc=yes"]
ret += ["--show-pid=no"]
boring_callers = common.BoringCallers(mangled=False, use_re_wildcards=False)
# TODO(timurrrr): In fact, we want "starting from .." instead of "below .."
for bc in boring_callers:
ret += ["--cut_stack_below=%s" % bc]
return ret
import sys
import time
from xml.dom.minidom import parse
from xml.parsers.expat import ExpatError
import common
# Global symbol table (yuck)
TheAddressTable = None
# These are regexps that define functions (using C++ mangled names)
# we don't want to see in stack traces while pretty printing
# or generating suppressions.
# Just stop printing the stack/suppression frames when the current one
# matches any of these.
_BORING_CALLERS = common.BoringCallers(mangled=True, use_re_wildcards=True)
def getTextOf(top_node, name):
''' Returns all text in all DOM nodes with a certain |name| that are children
of |top_node|.
'''
text = ""
for nodes_named in top_node.getElementsByTagName(name):
text += "".join([
node.data for node in nodes_named.childNodes
if node.nodeType == node.TEXT_NODE
])
return text
def ToolCommand(self):
"""Get the tool command to run."""
# WINHEAP is what Dr. Memory supports as there are issues w/ both
# jemalloc (http://code.google.com/p/drmemory/issues/detail?id=320) and
# tcmalloc (http://code.google.com/p/drmemory/issues/detail?id=314)
add_env = {
"CHROME_ALLOCATOR" : "WINHEAP",
"JSIMD_FORCEMMX" : "1", # http://code.google.com/p/drmemory/issues/detail?id=540
}
for k,v in add_env.iteritems():
logging.info("export %s=%s", k, v)
os.putenv(k, v)
drmem_cmd = os.getenv("DRMEMORY_COMMAND")
if not drmem_cmd:
raise RuntimeError, "Please set DRMEMORY_COMMAND environment variable " \
"with the path to drmemory.exe"
proc = drmem_cmd.split(" ")
# By default, don't run python (this will exclude python's children as well)
# to reduce runtime. We're not really interested in spending time finding
# bugs in the python implementation.
# With file-based config we must update the file every time, and
# it will affect simultaneous drmem uses by this user. While file-based
# config has many advantages, here we may want this-instance-only
# (http://code.google.com/p/drmemory/issues/detail?id=334).
drconfig_cmd = [ proc[0].replace("drmemory.exe", "drconfig.exe") ]
drconfig_cmd += ["-quiet"] # suppress errors about no 64-bit libs
run_drconfig = True
if self._options.follow_python:
logging.info("Following python children")
# -unreg fails if not already registered so query for that first
query_cmd = drconfig_cmd + ["-isreg", "python.exe"]
query_proc = subprocess.Popen(query_cmd, stdout=subprocess.PIPE,
shell=True)
(query_out, query_err) = query_proc.communicate()
if re.search("exe not registered", query_out):
run_drconfig = False # all set
else:
drconfig_cmd += ["-unreg", "python.exe"]
else:
logging.info("Excluding python children")
drconfig_cmd += ["-reg", "python.exe", "-norun"]
if run_drconfig:
drconfig_retcode = common.RunSubprocess(drconfig_cmd, self._timeout)
if drconfig_retcode:
logging.error("Configuring whether to follow python children failed " \
"with %d.", drconfig_retcode)
raise RuntimeError, "Configuring python children failed "
suppression_count = 0
supp_files = self._options.suppressions
if self.full_mode:
supp_files += [s.replace(".txt", "_full.txt") for s in supp_files]
for suppression_file in supp_files:
if os.path.exists(suppression_file):
suppression_count += 1
proc += ["-suppress", common.NormalizeWindowsPath(suppression_file)]
if not suppression_count:
logging.warning("WARNING: NOT USING SUPPRESSIONS!")
# Un-comment to dump Dr.Memory events on error
#proc += ["-dr_ops", "-dumpcore_mask", "-dr_ops", "0x8bff"]
# Un-comment and comment next line to debug Dr.Memory
#proc += ["-dr_ops", "-no_hide"]
#proc += ["-dr_ops", "-msgbox_mask", "-dr_ops", "15"]
#Proc += ["-dr_ops", "-stderr_mask", "-dr_ops", "15"]
# Ensure we see messages about Dr. Memory crashing!
proc += ["-dr_ops", "-stderr_mask", "-dr_ops", "12"]
if self._options.use_debug:
proc += ["-debug"]
proc += ["-logdir", common.NormalizeWindowsPath(self.log_dir)]
if self.log_parent_dir:
# gpu process on Windows Vista+ runs at Low Integrity and can only
# write to certain directories (http://crbug.com/119131)
symcache_dir = os.path.join(self.log_parent_dir, "drmemory.symcache")
elif self._options.build_dir:
# The other case is only possible with -t cmdline.
# Anyways, if we omit -symcache_dir the -logdir's value is used which
# should be fine.
symcache_dir = os.path.join(self._options.build_dir, "drmemory.symcache")
if symcache_dir:
if not os.path.exists(symcache_dir):
try:
os.mkdir(symcache_dir)
except OSError:
logging.warning("Can't create symcache dir?")
if os.path.exists(symcache_dir):
proc += ["-symcache_dir", common.NormalizeWindowsPath(symcache_dir)]
# Use -no_summary to suppress DrMemory's summary and init-time
# notifications. We generate our own with drmemory_analyze.py.
proc += ["-batch", "-no_summary"]
# Un-comment to disable interleaved output. Will also suppress error
# messages normally printed to stderr.
#proc += ["-quiet", "-no_results_to_stderr"]
proc += ["-callstack_max_frames", "40"]
# disable leak scan for now
proc += ["-no_count_leaks", "-no_leak_scan"]
# crbug.com/413215, no heap mismatch check for Windows release build binary
if common.IsWindows() and "Release" in self._options.build_dir:
proc += ["-no_check_delete_mismatch"]
# make callstacks easier to read
proc += ["-callstack_srcfile_prefix",
"build\\src,chromium\\src,crt_build\\self_x86"]
proc += ["-callstack_modname_hide",
"*drmemory*,chrome.dll"]
boring_callers = common.BoringCallers(mangled=False, use_re_wildcards=False)
# TODO(timurrrr): In fact, we want "starting from .." instead of "below .."
proc += ["-callstack_truncate_below", ",".join(boring_callers)]
if self.pattern_mode:
proc += ["-pattern", "0xf1fd", "-no_count_leaks", "-redzone_size", "0x20"]
elif not self.full_mode:
proc += ["-light"]
proc += self._tool_flags
# Dr.Memory requires -- to separate tool flags from the executable name.
proc += ["--"]
if self._options.indirect or self._options.indirect_webkit_layout:
# TODO(timurrrr): reuse for TSan on Windows
wrapper_path = os.path.join(self._source_dir,
"tools", "valgrind", "browser_wrapper_win.py")
wrapper = " ".join(["python", wrapper_path] + proc)
self.CreateBrowserWrapper(wrapper)
logging.info("browser wrapper = " + " ".join(proc))
if self._options.indirect_webkit_layout:
proc = self._args
# Layout tests want forward slashes.
wrapper = wrapper.replace('\\', '/')
proc += ["--wrapper", wrapper]
return proc
else:
proc = []
# Note that self._args begins with the name of the exe to be run.
self._args[0] = common.NormalizeWindowsPath(self._args[0])
proc += self._args
return proc
class HeapcheckWrapper(object):
TMP_FILE = 'heapcheck.log'
SANITY_TEST_SUPPRESSION = "Heapcheck sanity test"
LEAK_REPORT_RE = re.compile(
'Leak of ([0-9]*) bytes in ([0-9]*) objects allocated from:')
STACK_LINE_RE = re.compile('\s*@\s*(?:0x)?[0-9a-fA-F]+\s*([^\n]*)')
BORING_CALLERS = common.BoringCallers(mangled=False, use_re_wildcards=True)
def __init__(self, supp_files):
self._mode = 'strict'
self._timeout = 1200
self._nocleanup_on_exit = False
self._suppressions = []
for fname in supp_files:
self._suppressions.extend(
suppressions.ReadSuppressionsFromFile(fname))
if os.path.exists(self.TMP_FILE):
os.remove(self.TMP_FILE)
def PutEnvAndLog(self, env_name, env_value):
"""Sets the env var |env_name| to |env_value| and writes to logging.info.
"""
os.putenv(env_name, env_value)
logging.info('export %s=%s', env_name, env_value)
def Execute(self):
"""Executes the app to be tested."""
logging.info('starting execution...')
proc = ['sh', path_utils.ScriptDir() + '/heapcheck_std.sh']
proc += self._args
self.PutEnvAndLog('G_SLICE', 'always-malloc')
self.PutEnvAndLog('NSS_DISABLE_ARENA_FREE_LIST', '1')
self.PutEnvAndLog('NSS_DISABLE_UNLOAD', '1')
self.PutEnvAndLog('GTEST_DEATH_TEST_USE_FORK', '1')
self.PutEnvAndLog('HEAPCHECK', self._mode)
self.PutEnvAndLog('HEAP_CHECK_ERROR_EXIT_CODE', '0')
self.PutEnvAndLog('HEAP_CHECK_MAX_LEAKS', '-1')
self.PutEnvAndLog('KEEP_SHADOW_STACKS', '1')
self.PutEnvAndLog(
'PPROF_PATH',
path_utils.ScriptDir() +
'/../../third_party/tcmalloc/chromium/src/pprof')
self.PutEnvAndLog('LD_LIBRARY_PATH',
'/usr/lib/debug/:/usr/lib32/debug/')
# CHROME_DEVEL_SANDBOX causes problems with heapcheck
self.PutEnvAndLog('CHROME_DEVEL_SANDBOX', '')
return common.RunSubprocess(proc, self._timeout)
def Analyze(self, log_lines, check_sanity=False):
"""Analyzes the app's output and applies suppressions to the reports.
Analyze() searches the logs for leak reports and tries to apply
suppressions to them. Unsuppressed reports and other log messages are
dumped as is.
If |check_sanity| is True, the list of suppressed reports is searched for a
report starting with SANITY_TEST_SUPPRESSION. If there isn't one, Analyze
returns 2 regardless of the unsuppressed reports.
Args:
log_lines: An iterator over the app's log lines.
check_sanity: A flag that determines whether we should check the tool's
sanity.
Returns:
2, if the sanity check fails,
1, if unsuppressed reports remain in the output and the sanity check
passes,
0, if all the errors are suppressed and the sanity check passes.
"""
return_code = 0
# leak signature: [number of bytes, number of objects]
cur_leak_signature = None
cur_stack = []
cur_report = []
reported_hashes = {}
# Statistics grouped by suppression description:
# [hit count, bytes, objects].
used_suppressions = {}
for line in log_lines:
line = line.rstrip() # remove the trailing \n
match = self.STACK_LINE_RE.match(line)
if match:
cur_stack.append(match.groups()[0])
cur_report.append(line)
continue
else:
if cur_stack:
# Try to find the suppression that applies to the current leak stack.
description = ''
for supp in self._suppressions:
if supp.Match(cur_stack):
cur_stack = []
description = supp.description
break
if cur_stack:
if not cur_leak_signature:
print 'Missing leak signature for the following stack: '
for frame in cur_stack:
print ' ' + frame
print 'Aborting...'
return 3
# Drop boring callers from the stack to get less redundant info
# and fewer unique reports.
found_boring = False
for i in range(1, len(cur_stack)):
for j in self.BORING_CALLERS:
if re.match(j, cur_stack[i]):
cur_stack = cur_stack[:i]
cur_report = cur_report[:i]
found_boring = True
break
if found_boring:
break
error_hash = hash(
"".join(cur_stack)) & 0xffffffffffffffff
if error_hash not in reported_hashes:
reported_hashes[error_hash] = 1
# Print the report and set the return code to 1.
print(
'Leak of %d bytes in %d objects allocated from:'
% tuple(cur_leak_signature))
print '\n'.join(cur_report)
return_code = 1
# Generate the suppression iff the stack contains more than one
# frame (otherwise it's likely to be broken)
if len(cur_stack) > 1 or found_boring:
print '\nSuppression (error hash=#%016X#):\n{' % (
error_hash)
print ' <insert_a_suppression_name_here>'
print ' Heapcheck:Leak'
for frame in cur_stack:
print ' fun:' + frame
print '}\n\n'
else:
print(
'This stack may be broken due to omitted frame pointers.'
' It is not recommended to suppress it.\n')
else:
# Update the suppressions histogram.
if description in used_suppressions:
hits, bytes, objects = used_suppressions[
description]
hits += 1
bytes += cur_leak_signature[0]
objects += cur_leak_signature[1]
used_suppressions[description] = [
hits, bytes, objects
]
else:
used_suppressions[description] = [
1
] + cur_leak_signature
cur_stack = []
cur_report = []
cur_leak_signature = None
match = self.LEAK_REPORT_RE.match(line)
if match:
cur_leak_signature = map(int, match.groups())
else:
print line
# Print the list of suppressions used.
is_sane = False
if used_suppressions:
print
print '-----------------------------------------------------'
print 'Suppressions used:'
print ' count bytes objects name'
histo = {}
for description in used_suppressions:
if description.startswith(
HeapcheckWrapper.SANITY_TEST_SUPPRESSION):
is_sane = True
hits, bytes, objects = used_suppressions[description]
line = '%8d %8d %8d %s' % (hits, bytes, objects, description)
if hits in histo:
histo[hits].append(line)
else:
histo[hits] = [line]
keys = histo.keys()
keys.sort()
for count in keys:
for line in histo[count]:
print line
print '-----------------------------------------------------'
if check_sanity and not is_sane:
logging.error("Sanity check failed")
return 2
else:
return return_code
def RunTestsAndAnalyze(self, check_sanity):
exec_retcode = self.Execute()
log_file = file(self.TMP_FILE, 'r')
analyze_retcode = self.Analyze(log_file, check_sanity)
log_file.close()
if analyze_retcode:
logging.error("Analyze failed.")
return analyze_retcode
if exec_retcode:
logging.error("Test execution failed.")
return exec_retcode
else:
logging.info("Test execution completed successfully.")
return 0
def Main(self, args, check_sanity=False):
self._args = args
start = datetime.datetime.now()
retcode = -1
retcode = self.RunTestsAndAnalyze(check_sanity)
end = datetime.datetime.now()
seconds = (end - start).seconds
hours = seconds / 3600
seconds %= 3600
minutes = seconds / 60
seconds %= 60
logging.info('elapsed time: %02d:%02d:%02d', hours, minutes, seconds)
logging.info('For more information on the Heapcheck bot see '
'http://dev.chromium.org/developers/how-tos/'
'using-the-heap-leak-checker')
return retcode
