Esempio n. 1
0
def check_result_ignore(result, ignore_ranges):
    """
    Determines if the result has to be ignored.

    Any result will be ignored if its origin matches any bear names and its
    SourceRange overlaps with the ignore range.

    Note that everything after a space in the origin will be cut away, so the
    user can ignore results with an origin like `CSecurityBear (buffer)` with
    just `# Ignore CSecurityBear`.

    :param result:        The result that needs to be checked.
    :param ignore_ranges: A list of tuples, each containing a list of lower
                          cased affected bearnames and a SourceRange to
                          ignore. If any of the bearname lists is empty, it
                          is considered an ignore range for all bears.
                          This may be a list of globbed bear wildcards.
    :return:              True if the result has to be ignored.
    """
    for bears, range in ignore_ranges:
        orig = result.origin.lower().split(' ')[0]
        if (result.overlaps(range) and
                (len(bears) == 0 or orig in bears or fnmatch(orig, bears))):
            return True

    return False
Esempio n. 2
0
    def Analyze(self):
        """
        This method analyzes the document and sends back the result

        :return: The output is a list with an element for each section.
                 It contains:
                 - The name of the section
                 - Boolean which is true if all bears in the section executed
                   successfully
                 - List of results where each result is a list which contains:
                   (str)origin, (str)message, (str)file, (str)line_nr,
                   (str)severity
        """
        retval = []
        if self.path == "" or self.config_file == "":
            return retval

        args = ["--config=" + self.config_file]

        log_printer = ListLogPrinter()
        exitcode = 0
        try:
            yielded_results = False
            (sections,
             local_bears,
             global_bears,
             targets) = gather_configuration(fail_acquire_settings,
                                             log_printer,
                                             arg_list=args)

            for section_name in sections:
                section = sections[section_name]

                if not section.is_enabled(targets):
                    continue

                if any([fnmatch(self.path, file_pattern)
                        for file_pattern in path_list(section["files"])]):

                    section["files"].value = self.path
                    section_result = execute_section(
                        section=section,
                        global_bear_list=global_bears[section_name],
                        local_bear_list=local_bears[section_name],
                        print_results=lambda *args: True,
                        log_printer=log_printer,
                        file_diff_dict={})
                    yielded_results = yielded_results or section_result[0]

                    retval.append(
                        DbusDocument.results_to_dbus_struct(section_result,
                                                            section_name))

            if yielded_results:
                exitcode = 1
        except BaseException as exception:  # pylint: disable=broad-except
            exitcode = exitcode or get_exitcode(exception, log_printer)

        logs = [log.to_string_dict() for log in log_printer.logs]
        return (exitcode, logs, retval)
Esempio n. 3
0
 def extract_links_from_file(file, link_ignore_regex, link_ignore_list):
     link_ignore_regex = re.compile(link_ignore_regex)
     regex = re.compile(
         r"""
         ((git\+|bzr\+|svn\+|hg\+|)  # For VCS URLs
         https?://                   # http:// or https:// as only these
                                     # are supported by the ``requests``
                                     # library
         [^.:%\s_/?#[\]@\\]+         # Initial part of domain
         \.                          # A required dot `.`
         (
             ((?:%[A-Fa-f0-9][A-Fa-f0-9])*[^\s()%\'"`<>|\\\[\]]+)
                                     # Path name
                                     # This part allows precentage
                                     # encoding like %3F
                                     # and does not allow
                                     # any parenthesis: balanced or
                                     # unbalanced.
         |                           # OR
             \((?:%[A-Fa-f0-9][A-Fa-f0-9])*[^\s()%\'"`<>|\\\[\]]*\)
                                     # Path name contained within ()
                                     # This part allows path names that
                                     # are explicitly enclosed within one
                                     # set of parenthesis.
                                     # An example can be:
                                     # http://wik.org/Hello_(Adele_song)/200
         )
         *)
                                     # Thus, the whole part above
                                     # prevents matching of
                                     # Unbalanced parenthesis
         (?<!\.)(?<!,)               # Exclude trailing `.` or `,` from URL
         """, re.VERBOSE)
     file_context = {}
     for line_number, line in enumerate(file):
         xmlns_regex = re.compile(r'xmlns:?\w*="(.*)"')
         for match in re.findall(regex, line):
             link = match[0]
             link_context = file_context.get(link)
             if not link_context:
                 link_context = LINK_CONTEXT.no_context
                 xmlns_match = xmlns_regex.search(line)
                 if xmlns_match and link in xmlns_match.groups():
                     link_context |= LINK_CONTEXT.xml_namespace
                 if link.startswith(('hg+', 'bzr+', 'git+', 'svn+')):
                     link_context |= LINK_CONTEXT.pip_vcs_url
                 file_context[link] = link_context
             if not (link_ignore_regex.search(link) or
                     fnmatch(link, link_ignore_list)):
                 yield link, line_number, link_context
Esempio n. 4
0
def icollect(file_paths, ignored_globs=None):
    """
    Evaluate globs in file paths and return all matching files.

    :param file_paths:    file path or list of such that can include globs
    :param ignored_globs: list of globs to ignore when matching files
    :return:              iterator that yields tuple of path of a matching
                          file, the glob where it was found
    """
    if isinstance(file_paths, str):
        file_paths = [file_paths]

    for file_path in file_paths:
        for match in iglob(file_path):
            if not ignored_globs or not fnmatch(match, ignored_globs):
                yield match, file_path
Esempio n. 5
0
def remove_ignored(file_paths, ignored_globs):
    """
    Removes file paths from list if they are ignored.

    :param file_paths:    file path string or list of such
    :param ignored_globs: list of globs that match to-be-ignored file paths
    :return:              list without those items that should be ignored
    """
    file_paths = list(set(file_paths))

    for file_path in file_paths:
        for ignored_glob in ignored_globs:
            if fnmatch(file_path, ignored_glob):
                file_paths.remove(file_path)
                break
    return file_paths
 def find_links_in_file(file, network_timeout, link_ignore_regex,
                        link_ignore_list):
     link_ignore_regex = re.compile(link_ignore_regex)
     regex = re.compile(
         r"""
         ((git\+|bzr\+|svn\+|hg\+|)  # For VCS URLs
         https?://                   # http:// or https:// as only these
                                     # are supported by the ``requests``
                                     # library
         [^.:%\s_/?#[\]@\\]+         # Initial part of domain
         \.                          # A required dot `.`
         (
             (?:[^\s()%\'"`<>|\\\[\]]+)  # Path name
                                         # This part does not allow
                                         # any parenthesis: balanced or
                                         # unbalanced.
         |                               # OR
             \([^\s()%\'"`<>|\\\[\]]*\)  # Path name contained within ()
                                     # This part allows path names that
                                     # are explicitly enclosed within one
                                     # set of parenthesis.
                                     # An example can be:
                                     # http://wik.org/Hello_(Adele_song)/200
         )
         *)
                                     # Thus, the whole part above
                                     # prevents matching of
                                     # Unbalanced parenthesis
         (?<!\.)(?<!,)               # Exclude trailing `.` or `,` from URL
         """, re.VERBOSE)
     for line_number, line in enumerate(file):
         for match in re.findall(regex, line):
             link = match[0]
             if not (link_ignore_regex.search(link) or
                     fnmatch(link, link_ignore_list)):
                 if link.startswith(('hg+', 'bzr+', 'git+', 'svn+')):
                     link = InvalidLinkBear.parse_pip_vcs_url(link)
                 host = urlparse(link).netloc
                 code = InvalidLinkBear.get_status_code(
                            link,
                            network_timeout.get(host)
                            if host in network_timeout
                            else network_timeout.get('*')
                            if '*' in network_timeout
                            else InvalidLinkBear.DEFAULT_TIMEOUT)
                 yield line_number + 1, link, code
Esempio n. 7
0
def limit_paths(file_paths, limit_globs):
    """
    Limits file paths from list based on the given globs.

    :param file_paths:  file path string or list of such
    :param limit_globs: list of globs to limit the file paths by
    :return:            list with only those items that in the limited globs
    """
    file_paths = list(set(file_paths))
    limited_list = file_paths[:]

    for file_path in file_paths:
        for limit_glob in limit_globs:
            if not fnmatch(file_path, limit_glob):
                limited_list.remove(file_path)
                break

    return limited_list
Esempio n. 8
0
def check_result_ignore(result, ignore_ranges):
    """
    Determines if the result has to be ignored.

    :param result:        The result that needs to be checked.
    :param ignore_ranges: A list of tuples, each containing a list of lower
                          cased affected bearnames and a SourceRange to
                          ignore. If any of the bearname lists is empty, it
                          is considered an ignore range for all bears.
                          This may be a list of globbed bear wildcards.
    :return:              True if the result has to be ignored.
    """
    for bears, range in ignore_ranges:
        orig = result.origin.lower()
        if (result.overlaps(range) and
                (len(bears) == 0 or orig in bears or fnmatch(orig, bears))):
            return True

    return False
Esempio n. 9
0
def icollect(file_paths, ignored_globs=None, match_cache={}):
    """
    Evaluate globs in file paths and return all matching files.

    :param file_paths:    File path or list of such that can include globs
    :param ignored_globs: List of globs to ignore when matching files
    :param match_cache:   Dictionary to use for caching results
    :return:              Iterator that yields tuple of path of a matching
                          file, the glob where it was found
    """
    if isinstance(file_paths, str):
        file_paths = [file_paths]

    for file_path in file_paths:
        if file_path not in match_cache:
            match_cache[file_path] = list(iglob(file_path))

        for match in match_cache[file_path]:
            if not ignored_globs or not fnmatch(match, ignored_globs):
                yield match, file_path
Esempio n. 10
0
def autoapply_actions(results,
                      file_dict,
                      file_diff_dict,
                      section,
                      log_printer):
    """
    Auto-applies actions like defined in the given section.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param section:        The section.
    :param log_printer:    A log printer instance to log messages on.
    :return:               A list of unprocessed results.
    """

    default_actions, invalid_actions = get_default_actions(section)

    for bearname, actionname in invalid_actions.items():
        log_printer.warn('Selected default action {!r} for bear {!r} does '
                         'not exist. Ignoring action.'.format(actionname,
                                                              bearname))

    if len(default_actions) == 0:
        # There's nothing to auto-apply.
        return results

    not_processed_results = []
    for result in results:
        try:
            # Match full bear names deterministically, prioritized!
            action = default_actions[result.origin]
        except KeyError:
            for bear_glob in default_actions:
                if fnmatch(result.origin, bear_glob):
                    action = default_actions[bear_glob]
                    break
            else:
                not_processed_results.append(result)
                continue

        if not action.is_applicable(result, file_dict, file_diff_dict):
            log_printer.warn('Selected default action {!r} for bear {!r} is '
                             'not applicable. Action not applied.'.format(
                                 action.get_metadata().name, result.origin))
            not_processed_results.append(result)
            continue

        try:
            action().apply_from_section(result,
                                        file_dict,
                                        file_diff_dict,
                                        section)
            log_printer.info('Applied {!r} on {} from {!r}.'.format(
                action.get_metadata().name,
                result.location_repr(),
                result.origin))
        except Exception as ex:
            not_processed_results.append(result)
            log_printer.log_exception(
                'Failed to execute action {!r} with error: {}.'.format(
                    action.get_metadata().name, ex),
                ex)
            log_printer.debug('-> for result ' + repr(result) + '.')

    return not_processed_results
Esempio n. 11
0
 def _test_fnmatch(self, pattern, matches, non_matches):
     for match in matches:
         self.assertTrue(fnmatch(match, pattern))
     for non_match in non_matches:
         self.assertFalse(fnmatch(non_match, pattern))
Esempio n. 12
0
def autoapply_actions(results, file_dict, file_diff_dict, section,
                      log_printer):
    """
    Auto-applies actions like defined in the given section.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param section:        The section.
    :param log_printer:    A log printer instance to log messages on.
    :return:               A list of unprocessed results.
    """

    default_actions, invalid_actions = get_default_actions(section)

    for bearname, actionname in invalid_actions.items():
        log_printer.warn("Selected default action {!r} for bear {!r} does "
                         "not exist. Ignoring action.".format(
                             actionname, bearname))

    if len(default_actions) == 0:
        # There's nothing to auto-apply.
        return results

    not_processed_results = []
    for result in results:
        try:
            # Match full bear names deterministically, prioritized!
            action = default_actions[result.origin]
        except KeyError:
            for bear_glob in default_actions:
                if fnmatch(result.origin, bear_glob):
                    action = default_actions[bear_glob]
                    break
            else:
                not_processed_results.append(result)
                continue

        if not action.is_applicable(result, file_dict, file_diff_dict):
            log_printer.warn("Selected default action {!r} for bear {!r} is "
                             "not applicable. Action not applied.".format(
                                 action.get_metadata().name, result.origin))
            not_processed_results.append(result)
            continue

        try:
            action().apply_from_section(result, file_dict, file_diff_dict,
                                        section)
            log_printer.info("Applied {!r} on {} from {!r}.".format(
                action.get_metadata().name, result.location_repr(),
                result.origin))
        except Exception as ex:
            not_processed_results.append(result)
            log_printer.log_exception(
                "Failed to execute action {!r} with error: {}.".format(
                    action.get_metadata().name, ex), ex)
            log_printer.debug("-> for result " + repr(result) + ".")

    return not_processed_results
Esempio n. 13
0
 def _test_fnmatch(self, pattern, matches, non_matches):
     for match in matches:
         self.assertTrue(fnmatch(match, pattern))
     for non_match in non_matches:
         self.assertFalse(fnmatch(non_match, pattern))
Esempio n. 14
0
def autoapply_actions(results,
                      file_dict,
                      file_diff_dict,
                      section,
                      log_printer=None):
    """
    Auto-applies actions like defined in the given section.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param section:        The section.
    :param log_printer:    A log printer instance to log messages on.
    :return:               A list of unprocessed results.
    """
    bear_actions = []
    for result in results:
        bear_actions += result.actions
    default_actions, invalid_actions = get_default_actions(
        section, bear_actions)
    no_autoapply_warn = bool(section.get('no_autoapply_warn', False))
    for bearname, actionname in invalid_actions.items():
        logging.warning(f'Selected default action {actionname!r} '
                        f'for bear {bearname!r} does not '
                        'exist. Ignoring action.')

    if len(default_actions) == 0:
        # There's nothing to auto-apply.
        return results

    not_processed_results = []
    for result in results:
        try:
            # Match full bear names deterministically, prioritized!
            action = default_actions[result.origin]
        except KeyError:
            for bear_glob in default_actions:
                if fnmatch(result.origin, bear_glob):
                    action = default_actions[bear_glob]
                    break
            else:
                not_processed_results.append(result)
                continue
        if action not in bear_actions or action in result.actions:
            applicable = action.is_applicable(result, file_dict,
                                              file_diff_dict)
            if applicable is not True:
                if not no_autoapply_warn:
                    logging.warning(f'{result.origin}: {applicable}')
                not_processed_results.append(result)
                continue

            try:
                action.apply_from_section(result, file_dict, file_diff_dict,
                                          section)
                logging.info(
                    f'Applied {action.get_metadata().name!r} on '
                    f'{result.location_repr()} from {result.origin!r}.')
            except Exception as ex:
                not_processed_results.append(result)
                log_exception(
                    f'Failed to execute action {action.get_metadata().name!r} '
                    f'with error: {ex}.', ex)
                logging.debug('-> for result ' + repr(result) + '.')
        else:
            not_processed_results.append(result)

    return not_processed_results
Esempio n. 15
0
    def Analyze(self):
        """
        This method analyzes the document and sends back the result

        :return: The output is structure which has 3 items:
                 -  The exitcode from the analysis.
                 -  List of logs from the analysis.
                 -  List of information about each section that contains:

                    -  The name of the section.
                    -  Boolean which is true if all bears in the section
                       executed successfully.
                    -  List of results where each result is a string
                       dictionary which contains:
                       id, origin, message, file, line_nr, severity
        """
        retval = []
        if self.path == "" or self.config_file == "":
            return retval

        args = ["--config=" + self.config_file]

        log_printer = ListLogPrinter()
        exitcode = 0
        try:
            yielded_results = False
            (sections,
             local_bears,
             global_bears,
             targets) = gather_configuration(fail_acquire_settings,
                                             log_printer,
                                             arg_list=args)

            for section_name in sections:
                section = sections[section_name]

                if not section.is_enabled(targets):
                    continue

                if any([fnmatch(self.path, file_pattern)
                        for file_pattern in path_list(section["files"])]):

                    section["files"].value = self.path
                    section_result = execute_section(
                        section=section,
                        global_bear_list=global_bears[section_name],
                        local_bear_list=local_bears[section_name],
                        print_results=lambda *args: True,
                        log_printer=log_printer)
                    yielded_results = yielded_results or section_result[0]

                    retval.append(
                        DbusDocument.results_to_dbus_struct(section_result,
                                                            section_name))

            if yielded_results:
                exitcode = 1
        except BaseException as exception:  # pylint: disable=broad-except
            exitcode = exitcode or get_exitcode(exception, log_printer)

        logs = [log.to_string_dict() for log in log_printer.logs]
        return (exitcode, logs, retval)