def check_result_ignore(result, ignore_ranges): """ Determines if the result has to be ignored. Any result will be ignored if its origin matches any bear names and its SourceRange overlaps with the ignore range. Note that everything after a space in the origin will be cut away, so the user can ignore results with an origin like `CSecurityBear (buffer)` with just `# Ignore CSecurityBear`. :param result: The result that needs to be checked. :param ignore_ranges: A list of tuples, each containing a list of lower cased affected bearnames and a SourceRange to ignore. If any of the bearname lists is empty, it is considered an ignore range for all bears. This may be a list of globbed bear wildcards. :return: True if the result has to be ignored. """ for bears, range in ignore_ranges: orig = result.origin.lower().split(' ')[0] if (result.overlaps(range) and (len(bears) == 0 or orig in bears or fnmatch(orig, bears))): return True return False
def Analyze(self): """ This method analyzes the document and sends back the result :return: The output is a list with an element for each section. It contains: - The name of the section - Boolean which is true if all bears in the section executed successfully - List of results where each result is a list which contains: (str)origin, (str)message, (str)file, (str)line_nr, (str)severity """ retval = [] if self.path == "" or self.config_file == "": return retval args = ["--config=" + self.config_file] log_printer = ListLogPrinter() exitcode = 0 try: yielded_results = False (sections, local_bears, global_bears, targets) = gather_configuration(fail_acquire_settings, log_printer, arg_list=args) for section_name in sections: section = sections[section_name] if not section.is_enabled(targets): continue if any([fnmatch(self.path, file_pattern) for file_pattern in path_list(section["files"])]): section["files"].value = self.path section_result = execute_section( section=section, global_bear_list=global_bears[section_name], local_bear_list=local_bears[section_name], print_results=lambda *args: True, log_printer=log_printer, file_diff_dict={}) yielded_results = yielded_results or section_result[0] retval.append( DbusDocument.results_to_dbus_struct(section_result, section_name)) if yielded_results: exitcode = 1 except BaseException as exception: # pylint: disable=broad-except exitcode = exitcode or get_exitcode(exception, log_printer) logs = [log.to_string_dict() for log in log_printer.logs] return (exitcode, logs, retval)
def extract_links_from_file(file, link_ignore_regex, link_ignore_list): link_ignore_regex = re.compile(link_ignore_regex) regex = re.compile( r""" ((git\+|bzr\+|svn\+|hg\+|) # For VCS URLs https?:// # http:// or https:// as only these # are supported by the ``requests`` # library [^.:%\s_/?#[\]@\\]+ # Initial part of domain \. # A required dot `.` ( ((?:%[A-Fa-f0-9][A-Fa-f0-9])*[^\s()%\'"`<>|\\\[\]]+) # Path name # This part allows precentage # encoding like %3F # and does not allow # any parenthesis: balanced or # unbalanced. | # OR \((?:%[A-Fa-f0-9][A-Fa-f0-9])*[^\s()%\'"`<>|\\\[\]]*\) # Path name contained within () # This part allows path names that # are explicitly enclosed within one # set of parenthesis. # An example can be: # http://wik.org/Hello_(Adele_song)/200 ) *) # Thus, the whole part above # prevents matching of # Unbalanced parenthesis (?<!\.)(?<!,) # Exclude trailing `.` or `,` from URL """, re.VERBOSE) file_context = {} for line_number, line in enumerate(file): xmlns_regex = re.compile(r'xmlns:?\w*="(.*)"') for match in re.findall(regex, line): link = match[0] link_context = file_context.get(link) if not link_context: link_context = LINK_CONTEXT.no_context xmlns_match = xmlns_regex.search(line) if xmlns_match and link in xmlns_match.groups(): link_context |= LINK_CONTEXT.xml_namespace if link.startswith(('hg+', 'bzr+', 'git+', 'svn+')): link_context |= LINK_CONTEXT.pip_vcs_url file_context[link] = link_context if not (link_ignore_regex.search(link) or fnmatch(link, link_ignore_list)): yield link, line_number, link_context
def icollect(file_paths, ignored_globs=None): """ Evaluate globs in file paths and return all matching files. :param file_paths: file path or list of such that can include globs :param ignored_globs: list of globs to ignore when matching files :return: iterator that yields tuple of path of a matching file, the glob where it was found """ if isinstance(file_paths, str): file_paths = [file_paths] for file_path in file_paths: for match in iglob(file_path): if not ignored_globs or not fnmatch(match, ignored_globs): yield match, file_path
def remove_ignored(file_paths, ignored_globs): """ Removes file paths from list if they are ignored. :param file_paths: file path string or list of such :param ignored_globs: list of globs that match to-be-ignored file paths :return: list without those items that should be ignored """ file_paths = list(set(file_paths)) for file_path in file_paths: for ignored_glob in ignored_globs: if fnmatch(file_path, ignored_glob): file_paths.remove(file_path) break return file_paths
def find_links_in_file(file, network_timeout, link_ignore_regex, link_ignore_list): link_ignore_regex = re.compile(link_ignore_regex) regex = re.compile( r""" ((git\+|bzr\+|svn\+|hg\+|) # For VCS URLs https?:// # http:// or https:// as only these # are supported by the ``requests`` # library [^.:%\s_/?#[\]@\\]+ # Initial part of domain \. # A required dot `.` ( (?:[^\s()%\'"`<>|\\\[\]]+) # Path name # This part does not allow # any parenthesis: balanced or # unbalanced. | # OR \([^\s()%\'"`<>|\\\[\]]*\) # Path name contained within () # This part allows path names that # are explicitly enclosed within one # set of parenthesis. # An example can be: # http://wik.org/Hello_(Adele_song)/200 ) *) # Thus, the whole part above # prevents matching of # Unbalanced parenthesis (?<!\.)(?<!,) # Exclude trailing `.` or `,` from URL """, re.VERBOSE) for line_number, line in enumerate(file): for match in re.findall(regex, line): link = match[0] if not (link_ignore_regex.search(link) or fnmatch(link, link_ignore_list)): if link.startswith(('hg+', 'bzr+', 'git+', 'svn+')): link = InvalidLinkBear.parse_pip_vcs_url(link) host = urlparse(link).netloc code = InvalidLinkBear.get_status_code( link, network_timeout.get(host) if host in network_timeout else network_timeout.get('*') if '*' in network_timeout else InvalidLinkBear.DEFAULT_TIMEOUT) yield line_number + 1, link, code
def limit_paths(file_paths, limit_globs): """ Limits file paths from list based on the given globs. :param file_paths: file path string or list of such :param limit_globs: list of globs to limit the file paths by :return: list with only those items that in the limited globs """ file_paths = list(set(file_paths)) limited_list = file_paths[:] for file_path in file_paths: for limit_glob in limit_globs: if not fnmatch(file_path, limit_glob): limited_list.remove(file_path) break return limited_list
def check_result_ignore(result, ignore_ranges): """ Determines if the result has to be ignored. :param result: The result that needs to be checked. :param ignore_ranges: A list of tuples, each containing a list of lower cased affected bearnames and a SourceRange to ignore. If any of the bearname lists is empty, it is considered an ignore range for all bears. This may be a list of globbed bear wildcards. :return: True if the result has to be ignored. """ for bears, range in ignore_ranges: orig = result.origin.lower() if (result.overlaps(range) and (len(bears) == 0 or orig in bears or fnmatch(orig, bears))): return True return False
def icollect(file_paths, ignored_globs=None, match_cache={}): """ Evaluate globs in file paths and return all matching files. :param file_paths: File path or list of such that can include globs :param ignored_globs: List of globs to ignore when matching files :param match_cache: Dictionary to use for caching results :return: Iterator that yields tuple of path of a matching file, the glob where it was found """ if isinstance(file_paths, str): file_paths = [file_paths] for file_path in file_paths: if file_path not in match_cache: match_cache[file_path] = list(iglob(file_path)) for match in match_cache[file_path]: if not ignored_globs or not fnmatch(match, ignored_globs): yield match, file_path
def autoapply_actions(results, file_dict, file_diff_dict, section, log_printer): """ Auto-applies actions like defined in the given section. :param results: A list of results. :param file_dict: A dictionary containing the name of files and its contents. :param file_diff_dict: A dictionary that contains filenames as keys and diff objects as values. :param section: The section. :param log_printer: A log printer instance to log messages on. :return: A list of unprocessed results. """ default_actions, invalid_actions = get_default_actions(section) for bearname, actionname in invalid_actions.items(): log_printer.warn('Selected default action {!r} for bear {!r} does ' 'not exist. Ignoring action.'.format(actionname, bearname)) if len(default_actions) == 0: # There's nothing to auto-apply. return results not_processed_results = [] for result in results: try: # Match full bear names deterministically, prioritized! action = default_actions[result.origin] except KeyError: for bear_glob in default_actions: if fnmatch(result.origin, bear_glob): action = default_actions[bear_glob] break else: not_processed_results.append(result) continue if not action.is_applicable(result, file_dict, file_diff_dict): log_printer.warn('Selected default action {!r} for bear {!r} is ' 'not applicable. Action not applied.'.format( action.get_metadata().name, result.origin)) not_processed_results.append(result) continue try: action().apply_from_section(result, file_dict, file_diff_dict, section) log_printer.info('Applied {!r} on {} from {!r}.'.format( action.get_metadata().name, result.location_repr(), result.origin)) except Exception as ex: not_processed_results.append(result) log_printer.log_exception( 'Failed to execute action {!r} with error: {}.'.format( action.get_metadata().name, ex), ex) log_printer.debug('-> for result ' + repr(result) + '.') return not_processed_results
def _test_fnmatch(self, pattern, matches, non_matches): for match in matches: self.assertTrue(fnmatch(match, pattern)) for non_match in non_matches: self.assertFalse(fnmatch(non_match, pattern))
def autoapply_actions(results, file_dict, file_diff_dict, section, log_printer): """ Auto-applies actions like defined in the given section. :param results: A list of results. :param file_dict: A dictionary containing the name of files and its contents. :param file_diff_dict: A dictionary that contains filenames as keys and diff objects as values. :param section: The section. :param log_printer: A log printer instance to log messages on. :return: A list of unprocessed results. """ default_actions, invalid_actions = get_default_actions(section) for bearname, actionname in invalid_actions.items(): log_printer.warn("Selected default action {!r} for bear {!r} does " "not exist. Ignoring action.".format( actionname, bearname)) if len(default_actions) == 0: # There's nothing to auto-apply. return results not_processed_results = [] for result in results: try: # Match full bear names deterministically, prioritized! action = default_actions[result.origin] except KeyError: for bear_glob in default_actions: if fnmatch(result.origin, bear_glob): action = default_actions[bear_glob] break else: not_processed_results.append(result) continue if not action.is_applicable(result, file_dict, file_diff_dict): log_printer.warn("Selected default action {!r} for bear {!r} is " "not applicable. Action not applied.".format( action.get_metadata().name, result.origin)) not_processed_results.append(result) continue try: action().apply_from_section(result, file_dict, file_diff_dict, section) log_printer.info("Applied {!r} on {} from {!r}.".format( action.get_metadata().name, result.location_repr(), result.origin)) except Exception as ex: not_processed_results.append(result) log_printer.log_exception( "Failed to execute action {!r} with error: {}.".format( action.get_metadata().name, ex), ex) log_printer.debug("-> for result " + repr(result) + ".") return not_processed_results
def autoapply_actions(results, file_dict, file_diff_dict, section, log_printer=None): """ Auto-applies actions like defined in the given section. :param results: A list of results. :param file_dict: A dictionary containing the name of files and its contents. :param file_diff_dict: A dictionary that contains filenames as keys and diff objects as values. :param section: The section. :param log_printer: A log printer instance to log messages on. :return: A list of unprocessed results. """ bear_actions = [] for result in results: bear_actions += result.actions default_actions, invalid_actions = get_default_actions( section, bear_actions) no_autoapply_warn = bool(section.get('no_autoapply_warn', False)) for bearname, actionname in invalid_actions.items(): logging.warning(f'Selected default action {actionname!r} ' f'for bear {bearname!r} does not ' 'exist. Ignoring action.') if len(default_actions) == 0: # There's nothing to auto-apply. return results not_processed_results = [] for result in results: try: # Match full bear names deterministically, prioritized! action = default_actions[result.origin] except KeyError: for bear_glob in default_actions: if fnmatch(result.origin, bear_glob): action = default_actions[bear_glob] break else: not_processed_results.append(result) continue if action not in bear_actions or action in result.actions: applicable = action.is_applicable(result, file_dict, file_diff_dict) if applicable is not True: if not no_autoapply_warn: logging.warning(f'{result.origin}: {applicable}') not_processed_results.append(result) continue try: action.apply_from_section(result, file_dict, file_diff_dict, section) logging.info( f'Applied {action.get_metadata().name!r} on ' f'{result.location_repr()} from {result.origin!r}.') except Exception as ex: not_processed_results.append(result) log_exception( f'Failed to execute action {action.get_metadata().name!r} ' f'with error: {ex}.', ex) logging.debug('-> for result ' + repr(result) + '.') else: not_processed_results.append(result) return not_processed_results
def Analyze(self): """ This method analyzes the document and sends back the result :return: The output is structure which has 3 items: - The exitcode from the analysis. - List of logs from the analysis. - List of information about each section that contains: - The name of the section. - Boolean which is true if all bears in the section executed successfully. - List of results where each result is a string dictionary which contains: id, origin, message, file, line_nr, severity """ retval = [] if self.path == "" or self.config_file == "": return retval args = ["--config=" + self.config_file] log_printer = ListLogPrinter() exitcode = 0 try: yielded_results = False (sections, local_bears, global_bears, targets) = gather_configuration(fail_acquire_settings, log_printer, arg_list=args) for section_name in sections: section = sections[section_name] if not section.is_enabled(targets): continue if any([fnmatch(self.path, file_pattern) for file_pattern in path_list(section["files"])]): section["files"].value = self.path section_result = execute_section( section=section, global_bear_list=global_bears[section_name], local_bear_list=local_bears[section_name], print_results=lambda *args: True, log_printer=log_printer) yielded_results = yielded_results or section_result[0] retval.append( DbusDocument.results_to_dbus_struct(section_result, section_name)) if yielded_results: exitcode = 1 except BaseException as exception: # pylint: disable=broad-except exitcode = exitcode or get_exitcode(exception, log_printer) logs = [log.to_string_dict() for log in log_printer.logs] return (exitcode, logs, retval)