def check_result_ignore(result, ignore_ranges):
"""
Determines if the result has to be ignored.
Any result will be ignored if its origin matches any bear names and its
SourceRange overlaps with the ignore range.
Note that everything after a space in the origin will be cut away, so the
user can ignore results with an origin like `CSecurityBear (buffer)` with
just `# Ignore CSecurityBear`.
:param result: The result that needs to be checked.
:param ignore_ranges: A list of tuples, each containing a list of lower
cased affected bearnames and a SourceRange to
ignore. If any of the bearname lists is empty, it
is considered an ignore range for all bears.
This may be a list of globbed bear wildcards.
:return: True if the result has to be ignored.
"""
for bears, range in ignore_ranges:
orig = result.origin.lower().split(' ')[0]
if (result.overlaps(range) and
(len(bears) == 0 or orig in bears or fnmatch(orig, bears))):
return True
return False
def Analyze(self):
"""
This method analyzes the document and sends back the result
:return: The output is a list with an element for each section.
It contains:
- The name of the section
- Boolean which is true if all bears in the section executed
successfully
- List of results where each result is a list which contains:
(str)origin, (str)message, (str)file, (str)line_nr,
(str)severity
"""
retval = []
if self.path == "" or self.config_file == "":
return retval
args = ["--config=" + self.config_file]
log_printer = ListLogPrinter()
exitcode = 0
try:
yielded_results = False
(sections,
local_bears,
global_bears,
targets) = gather_configuration(fail_acquire_settings,
log_printer,
arg_list=args)
for section_name in sections:
section = sections[section_name]
if not section.is_enabled(targets):
continue
if any([fnmatch(self.path, file_pattern)
for file_pattern in path_list(section["files"])]):
section["files"].value = self.path
section_result = execute_section(
section=section,
global_bear_list=global_bears[section_name],
local_bear_list=local_bears[section_name],
print_results=lambda *args: True,
log_printer=log_printer,
file_diff_dict={})
yielded_results = yielded_results or section_result[0]
retval.append(
DbusDocument.results_to_dbus_struct(section_result,
section_name))
if yielded_results:
exitcode = 1
except BaseException as exception: # pylint: disable=broad-except
exitcode = exitcode or get_exitcode(exception, log_printer)
logs = [log.to_string_dict() for log in log_printer.logs]
return (exitcode, logs, retval)
def extract_links_from_file(file, link_ignore_regex, link_ignore_list):
link_ignore_regex = re.compile(link_ignore_regex)
regex = re.compile(
r"""
((git\+|bzr\+|svn\+|hg\+|) # For VCS URLs
https?:// # http:// or https:// as only these
# are supported by the ``requests``
# library
[^.:%\s_/?#[\]@\\]+ # Initial part of domain
\. # A required dot `.`
(
((?:%[A-Fa-f0-9][A-Fa-f0-9])*[^\s()%\'"`<>|\\\[\]]+)
# Path name
# This part allows precentage
# encoding like %3F
# and does not allow
# any parenthesis: balanced or
# unbalanced.
| # OR
\((?:%[A-Fa-f0-9][A-Fa-f0-9])*[^\s()%\'"`<>|\\\[\]]*\)
# Path name contained within ()
# This part allows path names that
# are explicitly enclosed within one
# set of parenthesis.
# An example can be:
# http://wik.org/Hello_(Adele_song)/200
)
*)
# Thus, the whole part above
# prevents matching of
# Unbalanced parenthesis
(?<!\.)(?<!,) # Exclude trailing `.` or `,` from URL
""", re.VERBOSE)
file_context = {}
for line_number, line in enumerate(file):
xmlns_regex = re.compile(r'xmlns:?\w*="(.*)"')
for match in re.findall(regex, line):
link = match[0]
link_context = file_context.get(link)
if not link_context:
link_context = LINK_CONTEXT.no_context
xmlns_match = xmlns_regex.search(line)
if xmlns_match and link in xmlns_match.groups():
link_context |= LINK_CONTEXT.xml_namespace
if link.startswith(('hg+', 'bzr+', 'git+', 'svn+')):
link_context |= LINK_CONTEXT.pip_vcs_url
file_context[link] = link_context
if not (link_ignore_regex.search(link) or
fnmatch(link, link_ignore_list)):
yield link, line_number, link_context
def icollect(file_paths, ignored_globs=None):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: file path or list of such that can include globs
:param ignored_globs: list of globs to ignore when matching files
:return: iterator that yields tuple of path of a matching
file, the glob where it was found
"""
if isinstance(file_paths, str):
file_paths = [file_paths]
for file_path in file_paths:
for match in iglob(file_path):
if not ignored_globs or not fnmatch(match, ignored_globs):
yield match, file_path
def remove_ignored(file_paths, ignored_globs):
"""
Removes file paths from list if they are ignored.
:param file_paths: file path string or list of such
:param ignored_globs: list of globs that match to-be-ignored file paths
:return: list without those items that should be ignored
"""
file_paths = list(set(file_paths))
for file_path in file_paths:
for ignored_glob in ignored_globs:
if fnmatch(file_path, ignored_glob):
file_paths.remove(file_path)
break
return file_paths
def find_links_in_file(file, network_timeout, link_ignore_regex,
link_ignore_list):
link_ignore_regex = re.compile(link_ignore_regex)
regex = re.compile(
r"""
((git\+|bzr\+|svn\+|hg\+|) # For VCS URLs
https?:// # http:// or https:// as only these
# are supported by the ``requests``
# library
[^.:%\s_/?#[\]@\\]+ # Initial part of domain
\. # A required dot `.`
(
(?:[^\s()%\'"`<>|\\\[\]]+) # Path name
# This part does not allow
# any parenthesis: balanced or
# unbalanced.
| # OR
\([^\s()%\'"`<>|\\\[\]]*\) # Path name contained within ()
# This part allows path names that
# are explicitly enclosed within one
# set of parenthesis.
# An example can be:
# http://wik.org/Hello_(Adele_song)/200
)
*)
# Thus, the whole part above
# prevents matching of
# Unbalanced parenthesis
(?<!\.)(?<!,) # Exclude trailing `.` or `,` from URL
""", re.VERBOSE)
for line_number, line in enumerate(file):
for match in re.findall(regex, line):
link = match[0]
if not (link_ignore_regex.search(link) or
fnmatch(link, link_ignore_list)):
if link.startswith(('hg+', 'bzr+', 'git+', 'svn+')):
link = InvalidLinkBear.parse_pip_vcs_url(link)
host = urlparse(link).netloc
code = InvalidLinkBear.get_status_code(
link,
network_timeout.get(host)
if host in network_timeout
else network_timeout.get('*')
if '*' in network_timeout
else InvalidLinkBear.DEFAULT_TIMEOUT)
yield line_number + 1, link, code
def limit_paths(file_paths, limit_globs):
"""
Limits file paths from list based on the given globs.
:param file_paths: file path string or list of such
:param limit_globs: list of globs to limit the file paths by
:return: list with only those items that in the limited globs
"""
file_paths = list(set(file_paths))
limited_list = file_paths[:]
for file_path in file_paths:
for limit_glob in limit_globs:
if not fnmatch(file_path, limit_glob):
limited_list.remove(file_path)
break
return limited_list
def check_result_ignore(result, ignore_ranges):
"""
Determines if the result has to be ignored.
:param result: The result that needs to be checked.
:param ignore_ranges: A list of tuples, each containing a list of lower
cased affected bearnames and a SourceRange to
ignore. If any of the bearname lists is empty, it
is considered an ignore range for all bears.
This may be a list of globbed bear wildcards.
:return: True if the result has to be ignored.
"""
for bears, range in ignore_ranges:
orig = result.origin.lower()
if (result.overlaps(range) and
(len(bears) == 0 or orig in bears or fnmatch(orig, bears))):
return True
return False
def icollect(file_paths, ignored_globs=None, match_cache={}):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: File path or list of such that can include globs
:param ignored_globs: List of globs to ignore when matching files
:param match_cache: Dictionary to use for caching results
:return: Iterator that yields tuple of path of a matching
file, the glob where it was found
"""
if isinstance(file_paths, str):
file_paths = [file_paths]
for file_path in file_paths:
if file_path not in match_cache:
match_cache[file_path] = list(iglob(file_path))
for match in match_cache[file_path]:
if not ignored_globs or not fnmatch(match, ignored_globs):
yield match, file_path
def autoapply_actions(results,
file_dict,
file_diff_dict,
section,
log_printer):
"""
Auto-applies actions like defined in the given section.
:param results: A list of results.
:param file_dict: A dictionary containing the name of files and its
contents.
:param file_diff_dict: A dictionary that contains filenames as keys and
diff objects as values.
:param section: The section.
:param log_printer: A log printer instance to log messages on.
:return: A list of unprocessed results.
"""
default_actions, invalid_actions = get_default_actions(section)
for bearname, actionname in invalid_actions.items():
log_printer.warn('Selected default action {!r} for bear {!r} does '
'not exist. Ignoring action.'.format(actionname,
bearname))
if len(default_actions) == 0:
# There's nothing to auto-apply.
return results
not_processed_results = []
for result in results:
try:
# Match full bear names deterministically, prioritized!
action = default_actions[result.origin]
except KeyError:
for bear_glob in default_actions:
if fnmatch(result.origin, bear_glob):
action = default_actions[bear_glob]
break
else:
not_processed_results.append(result)
continue
if not action.is_applicable(result, file_dict, file_diff_dict):
log_printer.warn('Selected default action {!r} for bear {!r} is '
'not applicable. Action not applied.'.format(
action.get_metadata().name, result.origin))
not_processed_results.append(result)
continue
try:
action().apply_from_section(result,
file_dict,
file_diff_dict,
section)
log_printer.info('Applied {!r} on {} from {!r}.'.format(
action.get_metadata().name,
result.location_repr(),
result.origin))
except Exception as ex:
not_processed_results.append(result)
log_printer.log_exception(
'Failed to execute action {!r} with error: {}.'.format(
action.get_metadata().name, ex),
ex)
log_printer.debug('-> for result ' + repr(result) + '.')
return not_processed_results
def _test_fnmatch(self, pattern, matches, non_matches):
for match in matches:
self.assertTrue(fnmatch(match, pattern))
for non_match in non_matches:
self.assertFalse(fnmatch(non_match, pattern))
def autoapply_actions(results, file_dict, file_diff_dict, section,
log_printer):
"""
Auto-applies actions like defined in the given section.
:param results: A list of results.
:param file_dict: A dictionary containing the name of files and its
contents.
:param file_diff_dict: A dictionary that contains filenames as keys and
diff objects as values.
:param section: The section.
:param log_printer: A log printer instance to log messages on.
:return: A list of unprocessed results.
"""
default_actions, invalid_actions = get_default_actions(section)
for bearname, actionname in invalid_actions.items():
log_printer.warn("Selected default action {!r} for bear {!r} does "
"not exist. Ignoring action.".format(
actionname, bearname))
if len(default_actions) == 0:
# There's nothing to auto-apply.
return results
not_processed_results = []
for result in results:
try:
# Match full bear names deterministically, prioritized!
action = default_actions[result.origin]
except KeyError:
for bear_glob in default_actions:
if fnmatch(result.origin, bear_glob):
action = default_actions[bear_glob]
break
else:
not_processed_results.append(result)
continue
if not action.is_applicable(result, file_dict, file_diff_dict):
log_printer.warn("Selected default action {!r} for bear {!r} is "
"not applicable. Action not applied.".format(
action.get_metadata().name, result.origin))
not_processed_results.append(result)
continue
try:
action().apply_from_section(result, file_dict, file_diff_dict,
section)
log_printer.info("Applied {!r} on {} from {!r}.".format(
action.get_metadata().name, result.location_repr(),
result.origin))
except Exception as ex:
not_processed_results.append(result)
log_printer.log_exception(
"Failed to execute action {!r} with error: {}.".format(
action.get_metadata().name, ex), ex)
log_printer.debug("-> for result " + repr(result) + ".")
return not_processed_results
def _test_fnmatch(self, pattern, matches, non_matches):
for match in matches:
self.assertTrue(fnmatch(match, pattern))
for non_match in non_matches:
self.assertFalse(fnmatch(non_match, pattern))
def autoapply_actions(results,
file_dict,
file_diff_dict,
section,
log_printer=None):
"""
Auto-applies actions like defined in the given section.
:param results: A list of results.
:param file_dict: A dictionary containing the name of files and its
contents.
:param file_diff_dict: A dictionary that contains filenames as keys and
diff objects as values.
:param section: The section.
:param log_printer: A log printer instance to log messages on.
:return: A list of unprocessed results.
"""
bear_actions = []
for result in results:
bear_actions += result.actions
default_actions, invalid_actions = get_default_actions(
section, bear_actions)
no_autoapply_warn = bool(section.get('no_autoapply_warn', False))
for bearname, actionname in invalid_actions.items():
logging.warning(f'Selected default action {actionname!r} '
f'for bear {bearname!r} does not '
'exist. Ignoring action.')
if len(default_actions) == 0:
# There's nothing to auto-apply.
return results
not_processed_results = []
for result in results:
try:
# Match full bear names deterministically, prioritized!
action = default_actions[result.origin]
except KeyError:
for bear_glob in default_actions:
if fnmatch(result.origin, bear_glob):
action = default_actions[bear_glob]
break
else:
not_processed_results.append(result)
continue
if action not in bear_actions or action in result.actions:
applicable = action.is_applicable(result, file_dict,
file_diff_dict)
if applicable is not True:
if not no_autoapply_warn:
logging.warning(f'{result.origin}: {applicable}')
not_processed_results.append(result)
continue
try:
action.apply_from_section(result, file_dict, file_diff_dict,
section)
logging.info(
f'Applied {action.get_metadata().name!r} on '
f'{result.location_repr()} from {result.origin!r}.')
except Exception as ex:
not_processed_results.append(result)
log_exception(
f'Failed to execute action {action.get_metadata().name!r} '
f'with error: {ex}.', ex)
logging.debug('-> for result ' + repr(result) + '.')
else:
not_processed_results.append(result)
return not_processed_results
def Analyze(self):
"""
This method analyzes the document and sends back the result
:return: The output is structure which has 3 items:
- The exitcode from the analysis.
- List of logs from the analysis.
- List of information about each section that contains:
- The name of the section.
- Boolean which is true if all bears in the section
executed successfully.
- List of results where each result is a string
dictionary which contains:
id, origin, message, file, line_nr, severity
"""
retval = []
if self.path == "" or self.config_file == "":
return retval
args = ["--config=" + self.config_file]
log_printer = ListLogPrinter()
exitcode = 0
try:
yielded_results = False
(sections,
local_bears,
global_bears,
targets) = gather_configuration(fail_acquire_settings,
log_printer,
arg_list=args)
for section_name in sections:
section = sections[section_name]
if not section.is_enabled(targets):
continue
if any([fnmatch(self.path, file_pattern)
for file_pattern in path_list(section["files"])]):
section["files"].value = self.path
section_result = execute_section(
section=section,
global_bear_list=global_bears[section_name],
local_bear_list=local_bears[section_name],
print_results=lambda *args: True,
log_printer=log_printer)
yielded_results = yielded_results or section_result[0]
retval.append(
DbusDocument.results_to_dbus_struct(section_result,
section_name))
if yielded_results:
exitcode = 1
except BaseException as exception: # pylint: disable=broad-except
exitcode = exitcode or get_exitcode(exception, log_printer)
logs = [log.to_string_dict() for log in log_printer.logs]
return (exitcode, logs, retval)