Example #1
0
 def _VerifyGitHashForAllComponents(self, deps):
     self.assertTrue(deps)
     self.assertTrue(isinstance(deps, dict))
     for component in deps.values():
         for key in ['revision', 'old_revision', 'new_revision']:
             if key in component:
                 self.assertTrue(utils.IsGitHash(component[key]))
Example #2
0
def GetRepositoryType(revision_number):
    """Returns the repository type of this revision number.

  Args:
    revision_number: A revision number or git hash.

  Returns:
    'git' or 'svn', depending on the revision_number.
  """
    if utils.IsGitHash(revision_number):
        return 'git'
    else:
        return 'svn'
    def FilterAndSortBlameList(self):
        """Filters and sorts the blame list."""
        # Sort the blame list by its position in stack.
        self.blame_list.sort(key=lambda blame: blame.stack_frame_index)

        filtered_blame_list = []

        for blame in self.blame_list:
            # If regression information is available, check if it needs to be
            # filtered.
            if blame.range_start and blame.range_end:

                # Discards results that are after the end of regression.
                if not utils.IsGitHash(blame.revision) and (int(
                        blame.range_end) <= int(blame.revision)):
                    continue

            filtered_blame_list.append(blame)

        self.blame_list = filtered_blame_list
Example #4
0
def FindMatchForStacktrace(stacktrace, components,
                           component_to_regression_dict):
  """Finds the culprit CL for stacktrace.

  The passed stacktrace is either from release build stacktrace
  or debug build stacktrace.

  Args:
    stacktrace: A list of parsed stacks within a stacktrace.
    components: A set of components to look for.
    component_to_regression_dict: A dictionary mapping component path to
                                  its regression.

  Returns:
    A list of match results from all stacks.
  """
  # A list to aggregate results from all the callstacks in the stacktrace.
  results = []
  results_lock = Lock()

  # Setup parsers.
  svn_parser = svn_repository_parser.SVNParser(CONFIG['svn'])
  git_parser = git_repository_parser.GitParser(component_to_regression_dict,
                                               CONFIG['git'])

  # Create a cache of parsed revisions.
  component_to_changelog_map = {}
  for component_path in components:
    component_object = component_to_regression_dict[component_path]
    range_start = component_object['old_revision']
    range_end = component_object['new_revision']

    # If range start is 0, the range is too large and the crash has been
    # introduced the archived build, so ignore this case.
    if range_start == '0':
      continue

    component_name = component_to_regression_dict[component_path]['name']

    is_git = utils.IsGitHash(range_start)
    if is_git:
      repository_parser = git_parser
    else:
      repository_parser = svn_parser

    (revisions, file_to_revision_map) = repository_parser.ParseChangelog(
        component_path, range_start, range_end)

    # If the returned map from ParseChangeLog is empty, we don't need to look
    # further because either the parsing failed or the changelog is empty.
    if not (revisions and file_to_revision_map):
      continue

    component_to_changelog_map[component_path] = (repository_parser,
                                                  component_name,
                                                  revisions,
                                                  file_to_revision_map)

  # Analyze each of the call stacks in the stacktrace.
  for callstack in stacktrace.stack_list:
    FindMatchForCallstack(callstack, components, component_to_changelog_map,
                          results, results_lock)

  return results
Example #5
0
def FindCulpritCLs(stacktrace_string,
                   build_type,
                   chrome_regression=None,
                   component_regression=None,
                   chrome_crash_revision=None,
                   component_crash_revision=None,
                   crashing_component_path=None,
                   crashing_component_name=None,
                   crashing_component_repo_url=None):
    """Returns the result, a list of result.Result objects and message.

  If either or both of component_regression and component_crash_revision is not
  None, is is assumed that crashing_component_path and
  crashing_component_repo_url are not None.

  Args:
    stacktrace_string: A string representing stacktrace.
    build_type: The type of the job.
    chrome_regression: A string, chrome regression from clusterfuzz, in format
                       '123456:123457'
    component_regression: A string, component regression in the same format.
    chrome_crash_revision: A crash revision of chrome, in string.
    component_crash_revision: A crash revision of the component,
                              if component build.
    crashing_component_path: A relative path of the crashing component, as in
                             DEPS file. For example, it would be 'src/v8' for
                             v8 and 'src/third_party/WebKit' for blink.
    crashing_component_name: A name of the crashing component, such as v8.
    crashing_component_repo_url: The URL of the crashing component's repo, as
                                 shown in DEPS file. For example,
                                 'https://chromium.googlesource.com/skia.git'
                                 for skia.

  Returns:
    A list of result objects, along with the short description on where the
    result is from.
  """
    build_type = build_type.lower()
    component_to_crash_revision_dict = {}
    component_to_regression_dict = {}

    # If chrome regression is available, parse DEPS file.
    chrome_regression = crash_utils.SplitRange(chrome_regression)
    if chrome_regression:
        chrome_regression_start = chrome_regression[0]
        chrome_regression_end = chrome_regression[1]

        # Do not parse regression information for crashes introduced before the
        # first archived build.
        if chrome_regression_start != '0':
            component_to_regression_dict = chromium_deps.GetChromiumComponentRange(
                chrome_regression_start, chrome_regression_end)
            if not component_to_regression_dict:
                return (
                    ('Failed to get component regression ranges for chromium '
                     'regression range %s:%s' %
                     (chrome_regression_start, chrome_regression_end)), [])

    # Parse crash revision.
    if chrome_crash_revision:
        component_to_crash_revision_dict = chromium_deps.GetChromiumComponents(
            chrome_crash_revision)
        if not component_to_crash_revision_dict:
            return ((
                'Failed to get component dependencies for chromium revision "%s"'
                % chrome_crash_revision), [])

    # Check if component regression information is available.
    component_regression = crash_utils.SplitRange(component_regression)
    if component_regression:
        component_regression_start = component_regression[0]
        component_regression_end = component_regression[1]

        # If this component already has an entry in parsed DEPS file, overwrite
        # regression range and url.
        if crashing_component_path in component_to_regression_dict:
            component_regression_info = \
                component_to_regression_dict[crashing_component_path]
            component_regression_info[
                'old_revision'] = component_regression_start
            component_regression_info[
                'new_revision'] = component_regression_end
            component_regression_info[
                'repository'] = crashing_component_repo_url

        # if this component does not have an entry, add the entry to the parsed
        # DEPS file.
        else:
            repository_type = crash_utils.GetRepositoryType(
                component_regression_start)
            component_regression_info = {
                'path': crashing_component_path,
                'rolled': True,
                'name': crashing_component_name,
                'old_revision': component_regression_start,
                'new_revision': component_regression_end,
                'repository': crashing_component_repo_url,
                'repository_type': repository_type
            }
            component_to_regression_dict[crashing_component_path] = \
                component_regression_info

    # If component crash revision is available, add it to the parsed crash
    # revisions.
    if component_crash_revision:

        # If this component has already a crash revision info, overwrite it.
        if crashing_component_path in component_to_crash_revision_dict:
            component_crash_revision_info = \
                component_to_crash_revision_dict[crashing_component_path]
            component_crash_revision_info[
                'revision'] = component_crash_revision
            component_crash_revision_info[
                'repository'] = crashing_component_repo_url

        # If not, add it to the parsed DEPS.
        else:
            if utils.IsGitHash(component_crash_revision):
                repository_type = 'git'
            else:
                repository_type = 'svn'
            component_crash_revision_info = {
                'path': crashing_component_path,
                'name': crashing_component_name,
                'repository': crashing_component_repo_url,
                'repository_type': repository_type,
                'revision': component_crash_revision
            }
            component_to_crash_revision_dict[crashing_component_path] = \
                component_crash_revision_info

    # Parsed DEPS is used to normalize the stacktrace. Since parsed regression
    # and parsed crash state essentially contain same information, use either.
    if component_to_regression_dict:
        parsed_deps = component_to_regression_dict
    elif component_to_crash_revision_dict:
        parsed_deps = component_to_crash_revision_dict
    else:
        return (('Identifying culprit CL requires at lease one of regression '
                 'information or crash revision'), [])

    # Split stacktrace into release build/debug build and parse them.
    (release_build_stacktrace,
     debug_build_stacktrace) = SplitStacktrace(stacktrace_string)
    if not (release_build_stacktrace or debug_build_stacktrace):
        parsed_release_build_stacktrace = stacktrace.Stacktrace(
            stacktrace_string.splitlines(), build_type, parsed_deps)
    else:
        parsed_release_build_stacktrace = stacktrace.Stacktrace(
            release_build_stacktrace, build_type, parsed_deps)

    parsed_debug_build_stacktrace = stacktrace.Stacktrace(
        debug_build_stacktrace, build_type, parsed_deps)

    # Get a highest priority callstack (main_stack) from stacktrace, with release
    # build stacktrace in higher priority than debug build stacktace. This stack
    # is the callstack to find blame information for.
    if parsed_release_build_stacktrace.stack_list:
        main_stack = parsed_release_build_stacktrace.GetCrashStack()
    elif parsed_debug_build_stacktrace.stack_list:
        main_stack = parsed_debug_build_stacktrace.GetCrashStack()
    else:
        if 'mac_' in build_type:
            return ('No line information available in stacktrace.', [])

        return (
            'Findit failed to find any stack trace. Is it in a new format?',
            [])

    # Run the algorithm on the parsed stacktrace, and return the result.
    stacktrace_list = [
        parsed_release_build_stacktrace, parsed_debug_build_stacktrace
    ]
    return findit.FindItForCrash(stacktrace_list, main_stack,
                                 component_to_regression_dict,
                                 component_to_crash_revision_dict)
def GetChromiumComponents(chromium_revision,
                          os_platform='unix',
                          deps_file_downloader=_GetContentOfDEPS):
    """Return a list of components used by Chrome of the given revision.

  Args:
    chromium_revision: Revision of the Chrome build: svn revision, or git hash.
    os_platform: The target platform of the Chrome build, eg. win, mac, etc.
    deps_file_downloader: A function that takes the chromium_revision as input,
                          and returns the content of the DEPS file. The returned
                          content is assumed to be trusted input and will be
                          evaluated as python code.

  Returns:
    A map from component path to parsed component name, repository URL,
    repository type and revision.
    Return None if an error occurs.
  """
    if os_platform.lower() == 'linux':
        os_platform = 'unix'

    chromium_git_base_url = CONFIG['chromium_git_base_url']

    if not utils.IsGitHash(chromium_revision):
        # Convert svn revision or commit position to Git hash.
        cr_rev_url_template = CONFIG['cr_rev_url']
        url = cr_rev_url_template % chromium_revision
        status_code, content = utils.GetHttpClient().Get(url,
                                                         timeout=120,
                                                         retries=5,
                                                         retry_if_not=404)
        if status_code != 200 or not content:
            if status_code == 404:
                print 'Chromium commit position %s is not found.' % chromium_revision
            return None

        cr_rev_data = json.loads(content)
        if 'git_sha' not in cr_rev_data:
            return None

        if 'repo' not in cr_rev_data or cr_rev_data['repo'] != 'chromium/src':
            print(
                '%s seems like a commit position of "%s", but not "chromium/src".'
                % (chromium_revision, cr_rev_data['repo']))
            return None

        chromium_revision = cr_rev_data.get('git_sha')
        if not chromium_revision:
            return None

    # Download the content of DEPS file in chromium.
    deps_content = deps_file_downloader(chromium_revision)
    if not deps_content:
        return None

    all_deps = {}

    # Parse the content of DEPS file.
    deps, deps_os = _ParseDEPS(deps_content)
    all_deps.update(deps)
    if os_platform is not None:
        all_deps.update(deps_os.get(os_platform, {}))

    # Figure out components based on the dependencies.
    components = {}
    host_dirs = CONFIG['host_directories']
    for component_path, component_repo_url in all_deps.iteritems():
        if component_repo_url is None:
            # For some platform like iso, some component is ignored.
            continue

        name = _GetComponentName(component_path, host_dirs)
        repository, revision = component_repo_url.split('@')
        match = OLD_GIT_URL_PATTERN.match(repository)
        if match:
            repository = 'https://chromium.googlesource.com/%s' % match.group(
                1)
        is_git_hash = utils.IsGitHash(revision)
        if is_git_hash:
            repository_type = 'git'
        else:
            repository_type = 'svn'
        if not component_path.endswith('/'):
            component_path += '/'
        components[component_path] = {
            'path': component_path,
            'name': name,
            'repository': repository,
            'repository_type': repository_type,
            'revision': revision
        }

    # Add chromium as a component.
    components['src/'] = {
        'path': 'src/',
        'name': 'chromium',
        'repository': chromium_git_base_url,
        'repository_type': 'git',
        'revision': chromium_revision
    }

    return components