def get_file_dict(filename_list, log_printer=None, allow_raw_files=False):
"""
Reads all files into a dictionary.
:param filename_list: List of names of paths to files to get contents of.
:param log_printer: The logger which logs errors.
:param allow_raw_files: Allow the usage of raw files (non text files),
disabled by default
:return: Reads the content of each file into a dictionary
with filenames as keys.
"""
file_dict = {}
for filename in filename_list:
try:
with open(filename, 'r',
encoding=detect_encoding(filename)) as _file:
file_dict[filename] = tuple(_file.readlines())
except UnicodeDecodeError:
if allow_raw_files:
file_dict[filename] = None
continue
logging.warning("Failed to read file '{}'. It seems to contain "
'non-unicode characters. Leaving it out.'
.format(filename))
except OSError as exception:
log_exception("Failed to read file '{}' because of an unknown "
'error. Leaving it out.'.format(filename),
exception,
log_level=LOG_LEVEL.WARNING)
logging.debug('Files that will be checked:\n' +
'\n'.join(file_dict.keys()))
return file_dict
def print_results_formatted(log_printer,
section,
result_list,
file_dict,
*args):
"""
Prints results through the format string from the format setting done by
user.
:param log_printer: Printer responsible for logging the messages.
:param section: The section to which the results belong.
:param result_list: List of Result objects containing the corresponding
results.
"""
default_format = ('id:{id}:origin:{origin}:file:{file}:line:{line}:'
'column:{column}:end_line:{end_line}:end_column:'
'{end_column}:severity:{severity}:severity_str:'
'{severity_str}:message:{message}')
format_str = str(section.get('format', default_format))
if format_str == 'True':
format_str = default_format
for result in result_list:
severity_str = RESULT_SEVERITY.__str__(result.severity)
format_args = vars(result)
try:
if len(result.affected_code) == 0:
format_args['affected_code'] = None
print(format_str.format(file=None,
line=None,
end_line=None,
column=None,
end_column=None,
severity_str=severity_str,
message=result.message,
**format_args))
continue
for range in result.affected_code:
format_args['affected_code'] = range
format_args['source_lines'] = range.affected_source(file_dict)
print(format_str.format(file=range.start.file,
line=range.start.line,
end_line=range.end.line,
column=range.start.column,
end_column=range.end.column,
severity_str=severity_str,
message=result.message,
**format_args))
except KeyError as exception:
log_exception(
'Unable to print the result with the given format string.',
exception)
def print_results_formatted(log_printer,
section,
result_list,
file_dict,
*args):
"""
Prints results through the format string from the format setting done by
user.
:param log_printer: Printer responsible for logging the messages.
:param section: The section to which the results belong.
:param result_list: List of Result objects containing the corresponding
results.
"""
default_format = ('id:{id}:origin:{origin}:file:{file}:line:{line}:'
'column:{column}:end_line:{end_line}:end_column:'
'{end_column}:severity:{severity}:severity_str:'
'{severity_str}:message:{message}')
format_str = str(section.get('format', default_format))
if format_str == 'True':
format_str = default_format
for result in result_list:
severity_str = RESULT_SEVERITY.__str__(result.severity)
format_args = vars(result)
try:
if len(result.affected_code) == 0:
format_args['affected_code'] = None
print(format_str.format(file=None,
line=None,
end_line=None,
column=None,
end_column=None,
severity_str=severity_str,
message=result.message,
**format_args))
continue
for range in result.affected_code:
format_args['affected_code'] = range
format_args['source_lines'] = range.affected_source(file_dict)
print(format_str.format(file=range.start.file,
line=range.start.line,
end_line=range.end.line,
column=range.start.column,
end_column=range.end.column,
severity_str=severity_str,
message=result.message,
**format_args))
except KeyError as exception:
log_exception(
'Unable to print the result with the given format string.',
exception)
def icollect_bears(bear_dir_glob, bear_globs, kinds, log_printer=None):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dir_glob: Directory globs or list of such that can contain bears
:param bear_globs: Globs of bears to collect
:param kinds: List of bear kinds to be collected
:param log_printer: Log_printer to handle logging
:return: Iterator that yields a tuple with bear class and
which bear_glob was used to find that bear class.
"""
for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]),
icollect(bear_dir_glob)):
# Since we get a real directory here and since we
# pass this later to iglob, we need to escape this.
bear_dir = glob_escape(bear_dir)
for bear_glob in bear_globs:
matching_files = iglob(os.path.join(bear_dir, bear_glob + '.py'))
matching_files = sorted(matching_files)
for matching_file in matching_files:
try:
for bear in _import_bears(matching_file, kinds):
yield bear, bear_glob
except pkg_resources.VersionConflict as exception:
log_exception(
(f'Unable to collect bears from {matching_file} '
'because there '
'is a conflict with the version of a dependency '
'you have installed. This may be resolved by '
'creating a separate virtual environment for coala '
f'or running `pip3 install \"{exception.req}\"`. '
'Be aware that '
'the latter solution might break other python '
'packages that depend on the currently installed '
'version.'),
exception,
log_level=LOG_LEVEL.WARNING)
except BaseException as exception:
log_exception(
f'Unable to collect bears from {matching_file}. '
'Probably the '
'file is malformed or the module code raises an '
'exception.',
exception,
log_level=LOG_LEVEL.WARNING)
def get_file_dict(self, filename_list, allow_raw_files=False):
"""
Builds a file dictionary from filename to lines of the file
from an associated FileProxyMap.
:param filename_list: List of files to get the contents of.
:param allow_raw_files: Allow the usage of raw files (non text files),
disabled by default
:return: Reads the content of each file into dictionary
with filenames as keys.
"""
if self.__proxymap is None:
raise ValueError('set_proxymap() should be called to set proxymap'
'of ProxyMapFileCache instance')
file_dict = {}
for filename in filename_list:
try:
# Keep the binary mode turned off to maintain compatibility
# with FileDictFileCache.get_file_dict().
# filename is assumed to be normcased.
proxy = self.__proxymap.resolve(filename,
hard_sync=True,
binary=False)
file_lines = proxy.lines()
file_dict[filename] = tuple(file_lines)
except UnicodeDecodeError:
if allow_raw_files:
file_dict[filename] = None
continue
logging.warning(f"Failed to read file '{filename}'. "
'It seems to contain '
'non-unicode characters. Leaving it out.')
except (OSError, ValueError) as exception:
log_exception(
f"Failed to read file '{filename}' "
'because of an unknown '
'error. Leaving it out.',
exception,
log_level=LOG_LEVEL.WARNING)
return file_dict
def icollect_bears(bear_dir_glob, bear_globs, kinds, log_printer=None):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dir_glob: Directory globs or list of such that can contain bears
:param bear_globs: Globs of bears to collect
:param kinds: List of bear kinds to be collected
:param log_printer: Log_printer to handle logging
:return: Iterator that yields a tuple with bear class and
which bear_glob was used to find that bear class.
"""
for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]),
icollect(bear_dir_glob)):
# Since we get a real directory here and since we
# pass this later to iglob, we need to escape this.
bear_dir = glob_escape(bear_dir)
for bear_glob in bear_globs:
matching_files = iglob(os.path.join(bear_dir, bear_glob + '.py'))
matching_files = sorted(matching_files)
for matching_file in matching_files:
try:
for bear in _import_bears(matching_file, kinds):
yield bear, bear_glob
except pkg_resources.VersionConflict as exception:
log_exception(
('Unable to collect bears from {file} because there '
'is a conflict with the version of a dependency '
'you have installed. This may be resolved by '
'creating a separate virtual environment for coala '
'or running `pip3 install \"{pkg}\"`. Be aware that '
'the latter solution might break other python '
'packages that depend on the currently installed '
'version.').format(file=matching_file,
pkg=exception.req),
exception, log_level=LOG_LEVEL.WARNING)
except BaseException as exception:
log_exception(
'Unable to collect bears from {file}. Probably the '
'file is malformed or the module code raises an '
'exception.'.format(file=matching_file),
exception,
log_level=LOG_LEVEL.WARNING)
def get_file_dict(self, filename_list, allow_raw_files=False):
"""
Builds a file dictionary from filename to lines of the file
from an associated FileProxyMap.
:param filename_list: List of files to get the contents of.
:param allow_raw_files: Allow the usage of raw files (non text files),
disabled by default
:return: Reads the content of each file into dictionary
with filenames as keys.
"""
if self.__proxymap is None:
raise ValueError('set_proxymap() should be called to set proxymap'
'of ProxyMapFileCache instance')
file_dict = {}
for filename in filename_list:
try:
# Keep the binary mode turned off to maintain compatibility
# with FileDictFileCache.get_file_dict().
# filename is assumed to be normcased.
proxy = self.__proxymap.resolve(filename,
hard_sync=True,
binary=False)
file_lines = proxy.lines()
file_dict[filename] = tuple(file_lines)
except UnicodeDecodeError:
if allow_raw_files:
file_dict[filename] = None
continue
logging.warning("Failed to read file '{}'. It seems to contain "
'non-unicode characters. Leaving it out.'
.format(filename))
except (OSError, ValueError) as exception:
log_exception("Failed to read file '{}' because of an unknown "
'error. Leaving it out.'.format(filename),
exception,
log_level=LOG_LEVEL.WARNING)
return file_dict
def get_file_dict(filename_list, log_printer=None, allow_raw_files=False):
"""
Reads all files into a dictionary.
:param filename_list: List of names of paths to files to get contents of.
:param log_printer: The logger which logs errors.
:param allow_raw_files: Allow the usage of raw files (non text files),
disabled by default
:return: Reads the content of each file into a dictionary
with filenames as keys.
"""
file_dict = FileDict()
for filename in filename_list:
try:
file_dict[filename] = FileFactory(filename)
# This is a workaround to purposely raise a
# ``UnicodeDecodeError`` to move into the
# except clause.
FileFactory(filename).string
except UnicodeDecodeError:
if allow_raw_files:
file_dict[filename] = None
continue
else:
# In case raw files are not allowed the
# ``FileFactory`` object created for it
# needs to be evicted from the dictionary.
del file_dict[filename]
logging.warning(
"Failed to read file '{}'. It seems to contain "
'non-unicode characters. Leaving it out.'.format(filename))
except OSError as exception:
log_exception("Failed to read file '{}' because of an unknown "
'error. Leaving it out.'.format(filename),
exception,
log_level=LOG_LEVEL.WARNING)
return file_dict
def get_file_dict(filename_list, log_printer=None, allow_raw_files=False):
"""
Reads all files into a dictionary.
:param filename_list: List of names of paths to files to get contents of.
:param log_printer: The logger which logs errors.
:param allow_raw_files: Allow the usage of raw files (non text files),
disabled by default
:return: Reads the content of each file into a dictionary
with filenames as keys.
"""
file_dict = FileDict()
for filename in filename_list:
try:
file_dict[filename] = File(filename)
# This is a workaround to purposely raise a
# ``UnicodeDecodeError`` to move into the
# except clause.
File(filename).string
except UnicodeDecodeError:
if allow_raw_files:
file_dict[filename] = None
continue
else:
# In case raw files are not allowed the
# ``File`` object created for it
# needs to be evicted from the dictionary.
del file_dict[filename]
logging.warning("Failed to read file '{}'. It seems to contain "
'non-unicode characters. Leaving it out.'
.format(filename))
except OSError as exception:
log_exception("Failed to read file '{}' because of an unknown "
'error. Leaving it out.'.format(filename),
exception,
log_level=LOG_LEVEL.WARNING)
return file_dict
def autoapply_actions(results,
file_dict,
file_diff_dict,
section,
log_printer=None):
"""
Auto-applies actions like defined in the given section.
:param results: A list of results.
:param file_dict: A dictionary containing the name of files and its
contents.
:param file_diff_dict: A dictionary that contains filenames as keys and
diff objects as values.
:param section: The section.
:param log_printer: A log printer instance to log messages on.
:return: A list of unprocessed results.
"""
default_actions, invalid_actions = get_default_actions(section)
no_autoapply_warn = bool(section.get('no_autoapply_warn', False))
for bearname, actionname in invalid_actions.items():
logging.warning('Selected default action {!r} for bear {!r} does not '
'exist. Ignoring action.'.format(actionname, bearname))
if len(default_actions) == 0:
# There's nothing to auto-apply.
return results
not_processed_results = []
for result in results:
try:
# Match full bear names deterministically, prioritized!
action = default_actions[result.origin]
except KeyError:
for bear_glob in default_actions:
if fnmatch(result.origin, bear_glob):
action = default_actions[bear_glob]
break
else:
not_processed_results.append(result)
continue
applicable = action.is_applicable(result, file_dict, file_diff_dict)
if applicable is not True:
if not no_autoapply_warn:
logging.warning('{}: {}'.format(result.origin, applicable))
not_processed_results.append(result)
continue
try:
action().apply_from_section(result, file_dict, file_diff_dict,
section)
logging.info('Applied {!r} on {} from {!r}.'.format(
action.get_metadata().name, result.location_repr(),
result.origin))
except Exception as ex:
not_processed_results.append(result)
log_exception(
'Failed to execute action {!r} with error: {}.'.format(
action.get_metadata().name, ex), ex)
logging.debug('-> for result ' + repr(result) + '.')
return not_processed_results
def autoapply_actions(results,
file_dict,
file_diff_dict,
section,
log_printer=None):
"""
Auto-applies actions like defined in the given section.
:param results: A list of results.
:param file_dict: A dictionary containing the name of files and its
contents.
:param file_diff_dict: A dictionary that contains filenames as keys and
diff objects as values.
:param section: The section.
:param log_printer: A log printer instance to log messages on.
:return: A list of unprocessed results.
"""
default_actions, invalid_actions = get_default_actions(section)
no_autoapply_warn = bool(section.get('no_autoapply_warn', False))
for bearname, actionname in invalid_actions.items():
logging.warning('Selected default action {!r} for bear {!r} does not '
'exist. Ignoring action.'.format(actionname, bearname))
if len(default_actions) == 0:
# There's nothing to auto-apply.
return results
not_processed_results = []
for result in results:
try:
# Match full bear names deterministically, prioritized!
action = default_actions[result.origin]
except KeyError:
for bear_glob in default_actions:
if fnmatch(result.origin, bear_glob):
action = default_actions[bear_glob]
break
else:
not_processed_results.append(result)
continue
applicable = action.is_applicable(result, file_dict, file_diff_dict)
if applicable is not True:
if not no_autoapply_warn:
logging.warning('{}: {}'.format(result.origin, applicable))
not_processed_results.append(result)
continue
try:
action().apply_from_section(result,
file_dict,
file_diff_dict,
section)
logging.info('Applied {!r} on {} from {!r}.'.format(
action.get_metadata().name,
result.location_repr(),
result.origin))
except Exception as ex:
not_processed_results.append(result)
log_exception(
'Failed to execute action {!r} with error: {}.'.format(
action.get_metadata().name, ex),
ex)
logging.debug('-> for result ' + repr(result) + '.')
return not_processed_results
