def _VerifyGitHashForAllComponents(self, deps):
self.assertTrue(deps)
self.assertTrue(isinstance(deps, dict))
for component in deps.values():
for key in ['revision', 'old_revision', 'new_revision']:
if key in component:
self.assertTrue(utils.IsGitHash(component[key]))
def GetRepositoryType(revision_number):
"""Returns the repository type of this revision number.
Args:
revision_number: A revision number or git hash.
Returns:
'git' or 'svn', depending on the revision_number.
"""
if utils.IsGitHash(revision_number):
return 'git'
else:
return 'svn'
def FilterAndSortBlameList(self):
"""Filters and sorts the blame list."""
# Sort the blame list by its position in stack.
self.blame_list.sort(key=lambda blame: blame.stack_frame_index)
filtered_blame_list = []
for blame in self.blame_list:
# If regression information is available, check if it needs to be
# filtered.
if blame.range_start and blame.range_end:
# Discards results that are after the end of regression.
if not utils.IsGitHash(blame.revision) and (int(
blame.range_end) <= int(blame.revision)):
continue
filtered_blame_list.append(blame)
self.blame_list = filtered_blame_list
def FindMatchForStacktrace(stacktrace, components,
component_to_regression_dict):
"""Finds the culprit CL for stacktrace.
The passed stacktrace is either from release build stacktrace
or debug build stacktrace.
Args:
stacktrace: A list of parsed stacks within a stacktrace.
components: A set of components to look for.
component_to_regression_dict: A dictionary mapping component path to
its regression.
Returns:
A list of match results from all stacks.
"""
# A list to aggregate results from all the callstacks in the stacktrace.
results = []
results_lock = Lock()
# Setup parsers.
svn_parser = svn_repository_parser.SVNParser(CONFIG['svn'])
git_parser = git_repository_parser.GitParser(component_to_regression_dict,
CONFIG['git'])
# Create a cache of parsed revisions.
component_to_changelog_map = {}
for component_path in components:
component_object = component_to_regression_dict[component_path]
range_start = component_object['old_revision']
range_end = component_object['new_revision']
# If range start is 0, the range is too large and the crash has been
# introduced the archived build, so ignore this case.
if range_start == '0':
continue
component_name = component_to_regression_dict[component_path]['name']
is_git = utils.IsGitHash(range_start)
if is_git:
repository_parser = git_parser
else:
repository_parser = svn_parser
(revisions, file_to_revision_map) = repository_parser.ParseChangelog(
component_path, range_start, range_end)
# If the returned map from ParseChangeLog is empty, we don't need to look
# further because either the parsing failed or the changelog is empty.
if not (revisions and file_to_revision_map):
continue
component_to_changelog_map[component_path] = (repository_parser,
component_name,
revisions,
file_to_revision_map)
# Analyze each of the call stacks in the stacktrace.
for callstack in stacktrace.stack_list:
FindMatchForCallstack(callstack, components, component_to_changelog_map,
results, results_lock)
return results
def FindCulpritCLs(stacktrace_string,
build_type,
chrome_regression=None,
component_regression=None,
chrome_crash_revision=None,
component_crash_revision=None,
crashing_component_path=None,
crashing_component_name=None,
crashing_component_repo_url=None):
"""Returns the result, a list of result.Result objects and message.
If either or both of component_regression and component_crash_revision is not
None, is is assumed that crashing_component_path and
crashing_component_repo_url are not None.
Args:
stacktrace_string: A string representing stacktrace.
build_type: The type of the job.
chrome_regression: A string, chrome regression from clusterfuzz, in format
'123456:123457'
component_regression: A string, component regression in the same format.
chrome_crash_revision: A crash revision of chrome, in string.
component_crash_revision: A crash revision of the component,
if component build.
crashing_component_path: A relative path of the crashing component, as in
DEPS file. For example, it would be 'src/v8' for
v8 and 'src/third_party/WebKit' for blink.
crashing_component_name: A name of the crashing component, such as v8.
crashing_component_repo_url: The URL of the crashing component's repo, as
shown in DEPS file. For example,
'https://chromium.googlesource.com/skia.git'
for skia.
Returns:
A list of result objects, along with the short description on where the
result is from.
"""
build_type = build_type.lower()
component_to_crash_revision_dict = {}
component_to_regression_dict = {}
# If chrome regression is available, parse DEPS file.
chrome_regression = crash_utils.SplitRange(chrome_regression)
if chrome_regression:
chrome_regression_start = chrome_regression[0]
chrome_regression_end = chrome_regression[1]
# Do not parse regression information for crashes introduced before the
# first archived build.
if chrome_regression_start != '0':
component_to_regression_dict = chromium_deps.GetChromiumComponentRange(
chrome_regression_start, chrome_regression_end)
if not component_to_regression_dict:
return (
('Failed to get component regression ranges for chromium '
'regression range %s:%s' %
(chrome_regression_start, chrome_regression_end)), [])
# Parse crash revision.
if chrome_crash_revision:
component_to_crash_revision_dict = chromium_deps.GetChromiumComponents(
chrome_crash_revision)
if not component_to_crash_revision_dict:
return ((
'Failed to get component dependencies for chromium revision "%s"'
% chrome_crash_revision), [])
# Check if component regression information is available.
component_regression = crash_utils.SplitRange(component_regression)
if component_regression:
component_regression_start = component_regression[0]
component_regression_end = component_regression[1]
# If this component already has an entry in parsed DEPS file, overwrite
# regression range and url.
if crashing_component_path in component_to_regression_dict:
component_regression_info = \
component_to_regression_dict[crashing_component_path]
component_regression_info[
'old_revision'] = component_regression_start
component_regression_info[
'new_revision'] = component_regression_end
component_regression_info[
'repository'] = crashing_component_repo_url
# if this component does not have an entry, add the entry to the parsed
# DEPS file.
else:
repository_type = crash_utils.GetRepositoryType(
component_regression_start)
component_regression_info = {
'path': crashing_component_path,
'rolled': True,
'name': crashing_component_name,
'old_revision': component_regression_start,
'new_revision': component_regression_end,
'repository': crashing_component_repo_url,
'repository_type': repository_type
}
component_to_regression_dict[crashing_component_path] = \
component_regression_info
# If component crash revision is available, add it to the parsed crash
# revisions.
if component_crash_revision:
# If this component has already a crash revision info, overwrite it.
if crashing_component_path in component_to_crash_revision_dict:
component_crash_revision_info = \
component_to_crash_revision_dict[crashing_component_path]
component_crash_revision_info[
'revision'] = component_crash_revision
component_crash_revision_info[
'repository'] = crashing_component_repo_url
# If not, add it to the parsed DEPS.
else:
if utils.IsGitHash(component_crash_revision):
repository_type = 'git'
else:
repository_type = 'svn'
component_crash_revision_info = {
'path': crashing_component_path,
'name': crashing_component_name,
'repository': crashing_component_repo_url,
'repository_type': repository_type,
'revision': component_crash_revision
}
component_to_crash_revision_dict[crashing_component_path] = \
component_crash_revision_info
# Parsed DEPS is used to normalize the stacktrace. Since parsed regression
# and parsed crash state essentially contain same information, use either.
if component_to_regression_dict:
parsed_deps = component_to_regression_dict
elif component_to_crash_revision_dict:
parsed_deps = component_to_crash_revision_dict
else:
return (('Identifying culprit CL requires at lease one of regression '
'information or crash revision'), [])
# Split stacktrace into release build/debug build and parse them.
(release_build_stacktrace,
debug_build_stacktrace) = SplitStacktrace(stacktrace_string)
if not (release_build_stacktrace or debug_build_stacktrace):
parsed_release_build_stacktrace = stacktrace.Stacktrace(
stacktrace_string.splitlines(), build_type, parsed_deps)
else:
parsed_release_build_stacktrace = stacktrace.Stacktrace(
release_build_stacktrace, build_type, parsed_deps)
parsed_debug_build_stacktrace = stacktrace.Stacktrace(
debug_build_stacktrace, build_type, parsed_deps)
# Get a highest priority callstack (main_stack) from stacktrace, with release
# build stacktrace in higher priority than debug build stacktace. This stack
# is the callstack to find blame information for.
if parsed_release_build_stacktrace.stack_list:
main_stack = parsed_release_build_stacktrace.GetCrashStack()
elif parsed_debug_build_stacktrace.stack_list:
main_stack = parsed_debug_build_stacktrace.GetCrashStack()
else:
if 'mac_' in build_type:
return ('No line information available in stacktrace.', [])
return (
'Findit failed to find any stack trace. Is it in a new format?',
[])
# Run the algorithm on the parsed stacktrace, and return the result.
stacktrace_list = [
parsed_release_build_stacktrace, parsed_debug_build_stacktrace
]
return findit.FindItForCrash(stacktrace_list, main_stack,
component_to_regression_dict,
component_to_crash_revision_dict)
def GetChromiumComponents(chromium_revision,
os_platform='unix',
deps_file_downloader=_GetContentOfDEPS):
"""Return a list of components used by Chrome of the given revision.
Args:
chromium_revision: Revision of the Chrome build: svn revision, or git hash.
os_platform: The target platform of the Chrome build, eg. win, mac, etc.
deps_file_downloader: A function that takes the chromium_revision as input,
and returns the content of the DEPS file. The returned
content is assumed to be trusted input and will be
evaluated as python code.
Returns:
A map from component path to parsed component name, repository URL,
repository type and revision.
Return None if an error occurs.
"""
if os_platform.lower() == 'linux':
os_platform = 'unix'
chromium_git_base_url = CONFIG['chromium_git_base_url']
if not utils.IsGitHash(chromium_revision):
# Convert svn revision or commit position to Git hash.
cr_rev_url_template = CONFIG['cr_rev_url']
url = cr_rev_url_template % chromium_revision
status_code, content = utils.GetHttpClient().Get(url,
timeout=120,
retries=5,
retry_if_not=404)
if status_code != 200 or not content:
if status_code == 404:
print 'Chromium commit position %s is not found.' % chromium_revision
return None
cr_rev_data = json.loads(content)
if 'git_sha' not in cr_rev_data:
return None
if 'repo' not in cr_rev_data or cr_rev_data['repo'] != 'chromium/src':
print(
'%s seems like a commit position of "%s", but not "chromium/src".'
% (chromium_revision, cr_rev_data['repo']))
return None
chromium_revision = cr_rev_data.get('git_sha')
if not chromium_revision:
return None
# Download the content of DEPS file in chromium.
deps_content = deps_file_downloader(chromium_revision)
if not deps_content:
return None
all_deps = {}
# Parse the content of DEPS file.
deps, deps_os = _ParseDEPS(deps_content)
all_deps.update(deps)
if os_platform is not None:
all_deps.update(deps_os.get(os_platform, {}))
# Figure out components based on the dependencies.
components = {}
host_dirs = CONFIG['host_directories']
for component_path, component_repo_url in all_deps.iteritems():
if component_repo_url is None:
# For some platform like iso, some component is ignored.
continue
name = _GetComponentName(component_path, host_dirs)
repository, revision = component_repo_url.split('@')
match = OLD_GIT_URL_PATTERN.match(repository)
if match:
repository = 'https://chromium.googlesource.com/%s' % match.group(
1)
is_git_hash = utils.IsGitHash(revision)
if is_git_hash:
repository_type = 'git'
else:
repository_type = 'svn'
if not component_path.endswith('/'):
component_path += '/'
components[component_path] = {
'path': component_path,
'name': name,
'repository': repository,
'repository_type': repository_type,
'revision': revision
}
# Add chromium as a component.
components['src/'] = {
'path': 'src/',
'name': 'chromium',
'repository': chromium_git_base_url,
'repository_type': 'git',
'revision': chromium_revision
}
return components