def _VerifyGitHashForAllComponents(self, deps): self.assertTrue(deps) self.assertTrue(isinstance(deps, dict)) for component in deps.values(): for key in ['revision', 'old_revision', 'new_revision']: if key in component: self.assertTrue(utils.IsGitHash(component[key]))
def GetRepositoryType(revision_number): """Returns the repository type of this revision number. Args: revision_number: A revision number or git hash. Returns: 'git' or 'svn', depending on the revision_number. """ if utils.IsGitHash(revision_number): return 'git' else: return 'svn'
def FilterAndSortBlameList(self): """Filters and sorts the blame list.""" # Sort the blame list by its position in stack. self.blame_list.sort(key=lambda blame: blame.stack_frame_index) filtered_blame_list = [] for blame in self.blame_list: # If regression information is available, check if it needs to be # filtered. if blame.range_start and blame.range_end: # Discards results that are after the end of regression. if not utils.IsGitHash(blame.revision) and (int( blame.range_end) <= int(blame.revision)): continue filtered_blame_list.append(blame) self.blame_list = filtered_blame_list
def FindMatchForStacktrace(stacktrace, components, component_to_regression_dict): """Finds the culprit CL for stacktrace. The passed stacktrace is either from release build stacktrace or debug build stacktrace. Args: stacktrace: A list of parsed stacks within a stacktrace. components: A set of components to look for. component_to_regression_dict: A dictionary mapping component path to its regression. Returns: A list of match results from all stacks. """ # A list to aggregate results from all the callstacks in the stacktrace. results = [] results_lock = Lock() # Setup parsers. svn_parser = svn_repository_parser.SVNParser(CONFIG['svn']) git_parser = git_repository_parser.GitParser(component_to_regression_dict, CONFIG['git']) # Create a cache of parsed revisions. component_to_changelog_map = {} for component_path in components: component_object = component_to_regression_dict[component_path] range_start = component_object['old_revision'] range_end = component_object['new_revision'] # If range start is 0, the range is too large and the crash has been # introduced the archived build, so ignore this case. if range_start == '0': continue component_name = component_to_regression_dict[component_path]['name'] is_git = utils.IsGitHash(range_start) if is_git: repository_parser = git_parser else: repository_parser = svn_parser (revisions, file_to_revision_map) = repository_parser.ParseChangelog( component_path, range_start, range_end) # If the returned map from ParseChangeLog is empty, we don't need to look # further because either the parsing failed or the changelog is empty. if not (revisions and file_to_revision_map): continue component_to_changelog_map[component_path] = (repository_parser, component_name, revisions, file_to_revision_map) # Analyze each of the call stacks in the stacktrace. for callstack in stacktrace.stack_list: FindMatchForCallstack(callstack, components, component_to_changelog_map, results, results_lock) return results
def FindCulpritCLs(stacktrace_string, build_type, chrome_regression=None, component_regression=None, chrome_crash_revision=None, component_crash_revision=None, crashing_component_path=None, crashing_component_name=None, crashing_component_repo_url=None): """Returns the result, a list of result.Result objects and message. If either or both of component_regression and component_crash_revision is not None, is is assumed that crashing_component_path and crashing_component_repo_url are not None. Args: stacktrace_string: A string representing stacktrace. build_type: The type of the job. chrome_regression: A string, chrome regression from clusterfuzz, in format '123456:123457' component_regression: A string, component regression in the same format. chrome_crash_revision: A crash revision of chrome, in string. component_crash_revision: A crash revision of the component, if component build. crashing_component_path: A relative path of the crashing component, as in DEPS file. For example, it would be 'src/v8' for v8 and 'src/third_party/WebKit' for blink. crashing_component_name: A name of the crashing component, such as v8. crashing_component_repo_url: The URL of the crashing component's repo, as shown in DEPS file. For example, 'https://chromium.googlesource.com/skia.git' for skia. Returns: A list of result objects, along with the short description on where the result is from. """ build_type = build_type.lower() component_to_crash_revision_dict = {} component_to_regression_dict = {} # If chrome regression is available, parse DEPS file. chrome_regression = crash_utils.SplitRange(chrome_regression) if chrome_regression: chrome_regression_start = chrome_regression[0] chrome_regression_end = chrome_regression[1] # Do not parse regression information for crashes introduced before the # first archived build. if chrome_regression_start != '0': component_to_regression_dict = chromium_deps.GetChromiumComponentRange( chrome_regression_start, chrome_regression_end) if not component_to_regression_dict: return ( ('Failed to get component regression ranges for chromium ' 'regression range %s:%s' % (chrome_regression_start, chrome_regression_end)), []) # Parse crash revision. if chrome_crash_revision: component_to_crash_revision_dict = chromium_deps.GetChromiumComponents( chrome_crash_revision) if not component_to_crash_revision_dict: return (( 'Failed to get component dependencies for chromium revision "%s"' % chrome_crash_revision), []) # Check if component regression information is available. component_regression = crash_utils.SplitRange(component_regression) if component_regression: component_regression_start = component_regression[0] component_regression_end = component_regression[1] # If this component already has an entry in parsed DEPS file, overwrite # regression range and url. if crashing_component_path in component_to_regression_dict: component_regression_info = \ component_to_regression_dict[crashing_component_path] component_regression_info[ 'old_revision'] = component_regression_start component_regression_info[ 'new_revision'] = component_regression_end component_regression_info[ 'repository'] = crashing_component_repo_url # if this component does not have an entry, add the entry to the parsed # DEPS file. else: repository_type = crash_utils.GetRepositoryType( component_regression_start) component_regression_info = { 'path': crashing_component_path, 'rolled': True, 'name': crashing_component_name, 'old_revision': component_regression_start, 'new_revision': component_regression_end, 'repository': crashing_component_repo_url, 'repository_type': repository_type } component_to_regression_dict[crashing_component_path] = \ component_regression_info # If component crash revision is available, add it to the parsed crash # revisions. if component_crash_revision: # If this component has already a crash revision info, overwrite it. if crashing_component_path in component_to_crash_revision_dict: component_crash_revision_info = \ component_to_crash_revision_dict[crashing_component_path] component_crash_revision_info[ 'revision'] = component_crash_revision component_crash_revision_info[ 'repository'] = crashing_component_repo_url # If not, add it to the parsed DEPS. else: if utils.IsGitHash(component_crash_revision): repository_type = 'git' else: repository_type = 'svn' component_crash_revision_info = { 'path': crashing_component_path, 'name': crashing_component_name, 'repository': crashing_component_repo_url, 'repository_type': repository_type, 'revision': component_crash_revision } component_to_crash_revision_dict[crashing_component_path] = \ component_crash_revision_info # Parsed DEPS is used to normalize the stacktrace. Since parsed regression # and parsed crash state essentially contain same information, use either. if component_to_regression_dict: parsed_deps = component_to_regression_dict elif component_to_crash_revision_dict: parsed_deps = component_to_crash_revision_dict else: return (('Identifying culprit CL requires at lease one of regression ' 'information or crash revision'), []) # Split stacktrace into release build/debug build and parse them. (release_build_stacktrace, debug_build_stacktrace) = SplitStacktrace(stacktrace_string) if not (release_build_stacktrace or debug_build_stacktrace): parsed_release_build_stacktrace = stacktrace.Stacktrace( stacktrace_string.splitlines(), build_type, parsed_deps) else: parsed_release_build_stacktrace = stacktrace.Stacktrace( release_build_stacktrace, build_type, parsed_deps) parsed_debug_build_stacktrace = stacktrace.Stacktrace( debug_build_stacktrace, build_type, parsed_deps) # Get a highest priority callstack (main_stack) from stacktrace, with release # build stacktrace in higher priority than debug build stacktace. This stack # is the callstack to find blame information for. if parsed_release_build_stacktrace.stack_list: main_stack = parsed_release_build_stacktrace.GetCrashStack() elif parsed_debug_build_stacktrace.stack_list: main_stack = parsed_debug_build_stacktrace.GetCrashStack() else: if 'mac_' in build_type: return ('No line information available in stacktrace.', []) return ( 'Findit failed to find any stack trace. Is it in a new format?', []) # Run the algorithm on the parsed stacktrace, and return the result. stacktrace_list = [ parsed_release_build_stacktrace, parsed_debug_build_stacktrace ] return findit.FindItForCrash(stacktrace_list, main_stack, component_to_regression_dict, component_to_crash_revision_dict)
def GetChromiumComponents(chromium_revision, os_platform='unix', deps_file_downloader=_GetContentOfDEPS): """Return a list of components used by Chrome of the given revision. Args: chromium_revision: Revision of the Chrome build: svn revision, or git hash. os_platform: The target platform of the Chrome build, eg. win, mac, etc. deps_file_downloader: A function that takes the chromium_revision as input, and returns the content of the DEPS file. The returned content is assumed to be trusted input and will be evaluated as python code. Returns: A map from component path to parsed component name, repository URL, repository type and revision. Return None if an error occurs. """ if os_platform.lower() == 'linux': os_platform = 'unix' chromium_git_base_url = CONFIG['chromium_git_base_url'] if not utils.IsGitHash(chromium_revision): # Convert svn revision or commit position to Git hash. cr_rev_url_template = CONFIG['cr_rev_url'] url = cr_rev_url_template % chromium_revision status_code, content = utils.GetHttpClient().Get(url, timeout=120, retries=5, retry_if_not=404) if status_code != 200 or not content: if status_code == 404: print 'Chromium commit position %s is not found.' % chromium_revision return None cr_rev_data = json.loads(content) if 'git_sha' not in cr_rev_data: return None if 'repo' not in cr_rev_data or cr_rev_data['repo'] != 'chromium/src': print( '%s seems like a commit position of "%s", but not "chromium/src".' % (chromium_revision, cr_rev_data['repo'])) return None chromium_revision = cr_rev_data.get('git_sha') if not chromium_revision: return None # Download the content of DEPS file in chromium. deps_content = deps_file_downloader(chromium_revision) if not deps_content: return None all_deps = {} # Parse the content of DEPS file. deps, deps_os = _ParseDEPS(deps_content) all_deps.update(deps) if os_platform is not None: all_deps.update(deps_os.get(os_platform, {})) # Figure out components based on the dependencies. components = {} host_dirs = CONFIG['host_directories'] for component_path, component_repo_url in all_deps.iteritems(): if component_repo_url is None: # For some platform like iso, some component is ignored. continue name = _GetComponentName(component_path, host_dirs) repository, revision = component_repo_url.split('@') match = OLD_GIT_URL_PATTERN.match(repository) if match: repository = 'https://chromium.googlesource.com/%s' % match.group( 1) is_git_hash = utils.IsGitHash(revision) if is_git_hash: repository_type = 'git' else: repository_type = 'svn' if not component_path.endswith('/'): component_path += '/' components[component_path] = { 'path': component_path, 'name': name, 'repository': repository, 'repository_type': repository_type, 'revision': revision } # Add chromium as a component. components['src/'] = { 'path': 'src/', 'name': 'chromium', 'repository': chromium_git_base_url, 'repository_type': 'git', 'revision': chromium_revision } return components