def get_file_dict(filename_list, log_printer=None, allow_raw_files=False): """ Reads all files into a dictionary. :param filename_list: List of names of paths to files to get contents of. :param log_printer: The logger which logs errors. :param allow_raw_files: Allow the usage of raw files (non text files), disabled by default :return: Reads the content of each file into a dictionary with filenames as keys. """ file_dict = {} for filename in filename_list: try: with open(filename, 'r', encoding=detect_encoding(filename)) as _file: file_dict[filename] = tuple(_file.readlines()) except UnicodeDecodeError: if allow_raw_files: file_dict[filename] = None continue logging.warning("Failed to read file '{}'. It seems to contain " 'non-unicode characters. Leaving it out.' .format(filename)) except OSError as exception: log_exception("Failed to read file '{}' because of an unknown " 'error. Leaving it out.'.format(filename), exception, log_level=LOG_LEVEL.WARNING) logging.debug('Files that will be checked:\n' + '\n'.join(file_dict.keys())) return file_dict
def print_results_formatted(log_printer, section, result_list, file_dict, *args): """ Prints results through the format string from the format setting done by user. :param log_printer: Printer responsible for logging the messages. :param section: The section to which the results belong. :param result_list: List of Result objects containing the corresponding results. """ default_format = ('id:{id}:origin:{origin}:file:{file}:line:{line}:' 'column:{column}:end_line:{end_line}:end_column:' '{end_column}:severity:{severity}:severity_str:' '{severity_str}:message:{message}') format_str = str(section.get('format', default_format)) if format_str == 'True': format_str = default_format for result in result_list: severity_str = RESULT_SEVERITY.__str__(result.severity) format_args = vars(result) try: if len(result.affected_code) == 0: format_args['affected_code'] = None print(format_str.format(file=None, line=None, end_line=None, column=None, end_column=None, severity_str=severity_str, message=result.message, **format_args)) continue for range in result.affected_code: format_args['affected_code'] = range format_args['source_lines'] = range.affected_source(file_dict) print(format_str.format(file=range.start.file, line=range.start.line, end_line=range.end.line, column=range.start.column, end_column=range.end.column, severity_str=severity_str, message=result.message, **format_args)) except KeyError as exception: log_exception( 'Unable to print the result with the given format string.', exception)
def icollect_bears(bear_dir_glob, bear_globs, kinds, log_printer=None): """ Collect all bears from bear directories that have a matching kind. :param bear_dir_glob: Directory globs or list of such that can contain bears :param bear_globs: Globs of bears to collect :param kinds: List of bear kinds to be collected :param log_printer: Log_printer to handle logging :return: Iterator that yields a tuple with bear class and which bear_glob was used to find that bear class. """ for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]), icollect(bear_dir_glob)): # Since we get a real directory here and since we # pass this later to iglob, we need to escape this. bear_dir = glob_escape(bear_dir) for bear_glob in bear_globs: matching_files = iglob(os.path.join(bear_dir, bear_glob + '.py')) matching_files = sorted(matching_files) for matching_file in matching_files: try: for bear in _import_bears(matching_file, kinds): yield bear, bear_glob except pkg_resources.VersionConflict as exception: log_exception( (f'Unable to collect bears from {matching_file} ' 'because there ' 'is a conflict with the version of a dependency ' 'you have installed. This may be resolved by ' 'creating a separate virtual environment for coala ' f'or running `pip3 install \"{exception.req}\"`. ' 'Be aware that ' 'the latter solution might break other python ' 'packages that depend on the currently installed ' 'version.'), exception, log_level=LOG_LEVEL.WARNING) except BaseException as exception: log_exception( f'Unable to collect bears from {matching_file}. ' 'Probably the ' 'file is malformed or the module code raises an ' 'exception.', exception, log_level=LOG_LEVEL.WARNING)
def get_file_dict(self, filename_list, allow_raw_files=False): """ Builds a file dictionary from filename to lines of the file from an associated FileProxyMap. :param filename_list: List of files to get the contents of. :param allow_raw_files: Allow the usage of raw files (non text files), disabled by default :return: Reads the content of each file into dictionary with filenames as keys. """ if self.__proxymap is None: raise ValueError('set_proxymap() should be called to set proxymap' 'of ProxyMapFileCache instance') file_dict = {} for filename in filename_list: try: # Keep the binary mode turned off to maintain compatibility # with FileDictFileCache.get_file_dict(). # filename is assumed to be normcased. proxy = self.__proxymap.resolve(filename, hard_sync=True, binary=False) file_lines = proxy.lines() file_dict[filename] = tuple(file_lines) except UnicodeDecodeError: if allow_raw_files: file_dict[filename] = None continue logging.warning(f"Failed to read file '{filename}'. " 'It seems to contain ' 'non-unicode characters. Leaving it out.') except (OSError, ValueError) as exception: log_exception( f"Failed to read file '{filename}' " 'because of an unknown ' 'error. Leaving it out.', exception, log_level=LOG_LEVEL.WARNING) return file_dict
def icollect_bears(bear_dir_glob, bear_globs, kinds, log_printer=None): """ Collect all bears from bear directories that have a matching kind. :param bear_dir_glob: Directory globs or list of such that can contain bears :param bear_globs: Globs of bears to collect :param kinds: List of bear kinds to be collected :param log_printer: Log_printer to handle logging :return: Iterator that yields a tuple with bear class and which bear_glob was used to find that bear class. """ for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]), icollect(bear_dir_glob)): # Since we get a real directory here and since we # pass this later to iglob, we need to escape this. bear_dir = glob_escape(bear_dir) for bear_glob in bear_globs: matching_files = iglob(os.path.join(bear_dir, bear_glob + '.py')) matching_files = sorted(matching_files) for matching_file in matching_files: try: for bear in _import_bears(matching_file, kinds): yield bear, bear_glob except pkg_resources.VersionConflict as exception: log_exception( ('Unable to collect bears from {file} because there ' 'is a conflict with the version of a dependency ' 'you have installed. This may be resolved by ' 'creating a separate virtual environment for coala ' 'or running `pip3 install \"{pkg}\"`. Be aware that ' 'the latter solution might break other python ' 'packages that depend on the currently installed ' 'version.').format(file=matching_file, pkg=exception.req), exception, log_level=LOG_LEVEL.WARNING) except BaseException as exception: log_exception( 'Unable to collect bears from {file}. Probably the ' 'file is malformed or the module code raises an ' 'exception.'.format(file=matching_file), exception, log_level=LOG_LEVEL.WARNING)
def get_file_dict(self, filename_list, allow_raw_files=False): """ Builds a file dictionary from filename to lines of the file from an associated FileProxyMap. :param filename_list: List of files to get the contents of. :param allow_raw_files: Allow the usage of raw files (non text files), disabled by default :return: Reads the content of each file into dictionary with filenames as keys. """ if self.__proxymap is None: raise ValueError('set_proxymap() should be called to set proxymap' 'of ProxyMapFileCache instance') file_dict = {} for filename in filename_list: try: # Keep the binary mode turned off to maintain compatibility # with FileDictFileCache.get_file_dict(). # filename is assumed to be normcased. proxy = self.__proxymap.resolve(filename, hard_sync=True, binary=False) file_lines = proxy.lines() file_dict[filename] = tuple(file_lines) except UnicodeDecodeError: if allow_raw_files: file_dict[filename] = None continue logging.warning("Failed to read file '{}'. It seems to contain " 'non-unicode characters. Leaving it out.' .format(filename)) except (OSError, ValueError) as exception: log_exception("Failed to read file '{}' because of an unknown " 'error. Leaving it out.'.format(filename), exception, log_level=LOG_LEVEL.WARNING) return file_dict
def get_file_dict(filename_list, log_printer=None, allow_raw_files=False): """ Reads all files into a dictionary. :param filename_list: List of names of paths to files to get contents of. :param log_printer: The logger which logs errors. :param allow_raw_files: Allow the usage of raw files (non text files), disabled by default :return: Reads the content of each file into a dictionary with filenames as keys. """ file_dict = FileDict() for filename in filename_list: try: file_dict[filename] = FileFactory(filename) # This is a workaround to purposely raise a # ``UnicodeDecodeError`` to move into the # except clause. FileFactory(filename).string except UnicodeDecodeError: if allow_raw_files: file_dict[filename] = None continue else: # In case raw files are not allowed the # ``FileFactory`` object created for it # needs to be evicted from the dictionary. del file_dict[filename] logging.warning( "Failed to read file '{}'. It seems to contain " 'non-unicode characters. Leaving it out.'.format(filename)) except OSError as exception: log_exception("Failed to read file '{}' because of an unknown " 'error. Leaving it out.'.format(filename), exception, log_level=LOG_LEVEL.WARNING) return file_dict
def get_file_dict(filename_list, log_printer=None, allow_raw_files=False): """ Reads all files into a dictionary. :param filename_list: List of names of paths to files to get contents of. :param log_printer: The logger which logs errors. :param allow_raw_files: Allow the usage of raw files (non text files), disabled by default :return: Reads the content of each file into a dictionary with filenames as keys. """ file_dict = FileDict() for filename in filename_list: try: file_dict[filename] = File(filename) # This is a workaround to purposely raise a # ``UnicodeDecodeError`` to move into the # except clause. File(filename).string except UnicodeDecodeError: if allow_raw_files: file_dict[filename] = None continue else: # In case raw files are not allowed the # ``File`` object created for it # needs to be evicted from the dictionary. del file_dict[filename] logging.warning("Failed to read file '{}'. It seems to contain " 'non-unicode characters. Leaving it out.' .format(filename)) except OSError as exception: log_exception("Failed to read file '{}' because of an unknown " 'error. Leaving it out.'.format(filename), exception, log_level=LOG_LEVEL.WARNING) return file_dict
def autoapply_actions(results, file_dict, file_diff_dict, section, log_printer=None): """ Auto-applies actions like defined in the given section. :param results: A list of results. :param file_dict: A dictionary containing the name of files and its contents. :param file_diff_dict: A dictionary that contains filenames as keys and diff objects as values. :param section: The section. :param log_printer: A log printer instance to log messages on. :return: A list of unprocessed results. """ default_actions, invalid_actions = get_default_actions(section) no_autoapply_warn = bool(section.get('no_autoapply_warn', False)) for bearname, actionname in invalid_actions.items(): logging.warning('Selected default action {!r} for bear {!r} does not ' 'exist. Ignoring action.'.format(actionname, bearname)) if len(default_actions) == 0: # There's nothing to auto-apply. return results not_processed_results = [] for result in results: try: # Match full bear names deterministically, prioritized! action = default_actions[result.origin] except KeyError: for bear_glob in default_actions: if fnmatch(result.origin, bear_glob): action = default_actions[bear_glob] break else: not_processed_results.append(result) continue applicable = action.is_applicable(result, file_dict, file_diff_dict) if applicable is not True: if not no_autoapply_warn: logging.warning('{}: {}'.format(result.origin, applicable)) not_processed_results.append(result) continue try: action().apply_from_section(result, file_dict, file_diff_dict, section) logging.info('Applied {!r} on {} from {!r}.'.format( action.get_metadata().name, result.location_repr(), result.origin)) except Exception as ex: not_processed_results.append(result) log_exception( 'Failed to execute action {!r} with error: {}.'.format( action.get_metadata().name, ex), ex) logging.debug('-> for result ' + repr(result) + '.') return not_processed_results