def test_equality(self): a = ['first', 'second', 'third'] b = ['first', 'third'] diff_1 = Diff.from_string_arrays(a, b) c = ['first', 'second', 'third'] d = ['first', 'third'] diff_2 = Diff.from_string_arrays(c, d) self.assertEqual(diff_1, diff_2) # changing the original array should not influence # the diff a[1] = 'else' self.assertEqual(diff_1, diff_2) diff_1.rename = 'abcd' self.assertNotEqual(diff_1, diff_2) diff_1.rename = False diff_1.delete = True self.assertNotEqual(diff_1, diff_2) diff_1.delete = False diff_1.add_lines(1, ['1']) self.assertNotEqual(diff_1, diff_2)
def test_from_string_arrays(self): a = ['q', 'a', 'b', 'x', 'c', 'd'] b = ['a', 'b', 'y', 'c', 'd', 'f'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first', 'fourth'] b = ['first', 'second', 'third', 'fourth'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first', 'fourth'] b = ['first_changed', 'second', 'third', 'fourth'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first', 'second', 'third', 'fourth'] b = ['first', 'fourth'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first', 'second', 'third', 'fourth'] b = ['first_changed', 'second_changed', 'fourth'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b)
def test_from_string_arrays(self): a = ['q\n', 'a\n', 'b\n', 'x\n', 'c\n', 'd\n'] b = ['a\n', 'b\n', 'y\n', 'c\n', 'd\n', 'f\n'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first\n', 'fourth\n'] b = ['first\n', 'second\n', 'third\n', 'fourth\n'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first\n', 'fourth\n'] b = ['first_changed\n', 'second\n', 'third\n', 'fourth\n'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first\n', 'second\n', 'third\n', 'fourth\n'] b = ['first\n', 'fourth\n'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first\n', 'second\n', 'third\n', 'fourth\n'] b = ['first_changed\n', 'second_changed\n', 'fourth\n'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b)
def test_equality(self): a = ['first', 'second', 'third'] b = ['first', 'third'] diff_1 = Diff.from_string_arrays(a, b) c = ['first', 'second', 'third'] d = ['first', 'third'] diff_2 = Diff.from_string_arrays(c, d) self.assertEqual(diff_1, diff_2) # changing the original array should not influence # the diff a[1] = 'else' self.assertEqual(diff_1, diff_2) diff_1.rename = 'abcd' self.assertNotEqual(diff_1, diff_2) diff_1.rename = False diff_1.delete = True self.assertNotEqual(diff_1, diff_2) diff_1.delete = False diff_1.add_lines(1, ['1']) self.assertNotEqual(diff_1, diff_2)
def test_from_string_arrays(self): a = ['q\n', 'a\n', 'b\n', 'x\n', 'c\n', 'd\n'] b = ['a\n', 'b\n', 'y\n', 'c\n', 'd\n', 'f\n'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first\n', 'fourth\n'] b = ['first\n', 'second\n', 'third\n', 'fourth\n'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first\n', 'fourth\n'] b = ['first_changed\n', 'second\n', 'third\n', 'fourth\n'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first\n', 'second\n', 'third\n', 'fourth\n'] b = ['first\n', 'fourth\n'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first\n', 'second\n', 'third\n', 'fourth\n'] b = ['first_changed\n', 'second_changed\n', 'fourth\n'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b)
def test_from_string_arrays(self): a = ["q", "a", "b", "x", "c", "d"] b = ["a", "b", "y", "c", "d", "f"] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ["first", "fourth"] b = ["first", "second", "third", "fourth"] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ["first", "fourth"] b = ["first_changed", "second", "third", "fourth"] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ["first", "second", "third", "fourth"] b = ["first", "fourth"] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ["first", "second", "third", "fourth"] b = ["first_changed", "second_changed", "fourth"] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b)
def test_from_string_arrays(self): a = ['q', 'a', 'b', 'x', 'c', 'd'] b = ['a', 'b', 'y', 'c', 'd', 'f'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first', 'fourth'] b = ['first', 'second', 'third', 'fourth'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first', 'fourth'] b = ['first_changed', 'second', 'third', 'fourth'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first', 'second', 'third', 'fourth'] b = ['first', 'fourth'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b) a = ['first', 'second', 'third', 'fourth'] b = ['first_changed', 'second_changed', 'fourth'] self.uut = Diff.from_string_arrays(a, b) self.assertEqual(self.uut.modified, b)
def test_equality(self): a = ["first", "second", "third"] b = ["first", "third"] diff_1 = Diff.from_string_arrays(a, b) a[1] = "else" diff_2 = Diff.from_string_arrays(a, b) self.assertEqual(diff_1, diff_2) diff_1.add_lines(1, ["1"]) self.assertNotEqual(diff_1, diff_2)
def remove_result_ranges_diffs(result_list, file_dict): """ Calculates the diffs to all files in file_dict that describe the removal of each respective result's affected code. :param result_list: list of results :param file_dict: dict of file contents :return: returnvalue[result][file] is a diff of the changes the removal of this result's affected code would cause for the file. """ result_diff_dict_dict = {} for original_result in result_list: mod_file_dict = copy.deepcopy(file_dict) for source_range in reversed(original_result.affected_code): file_name = source_range.file new_file = remove_range(mod_file_dict[file_name], source_range) mod_file_dict[file_name] = new_file diff_dict = {} for file_name in file_dict: diff_dict[file_name] = Diff.from_string_arrays( file_dict[file_name], mod_file_dict[file_name]) result_diff_dict_dict[original_result] = diff_dict return result_diff_dict_dict
def test_process_output_corrected(self): uut = (linter(sys.executable, output_format="corrected") (self.EmptyTestLinter) (self.section, None)) original = ["void main() {\n", "return 09;\n", "}\n"] fixed = ["void main()\n", "{\n", "return 9;\n", "}\n"] fixed_string = "".join(fixed) results = list(uut.process_output(fixed_string, "some-file.c", original)) diffs = list(Diff.from_string_arrays(original, fixed).split_diff()) expected = [Result.from_values(uut, "Inconsistency found.", "some-file.c", 1, None, 2, None, RESULT_SEVERITY.NORMAL, diffs={"some-file.c": diffs[0]})] self.assertEqual(results, expected) # Test when providing a sequence as output. results = list(uut.process_output([fixed_string, fixed_string], "some-file.c", original)) self.assertEqual(results, 2 * expected)
def run( self, filename, file, json_sort: bool = False, indent_size: int = SpacingHelper.DEFAULT_TAB_WIDTH, escape_unicode: bool = True, ): """ Raises issues for any deviations from the pretty-printed JSON. :param json_sort: Whether or not keys should be sorted. :param indent_size: Number of spaces per indentation level. :param escape_unicode: Whether or not to escape unicode values using ASCII. """ # Output a meaningful message if empty file given as input if len(file) == 0: yield Result.from_values(self, 'This file is empty.', file=filename) return try: json_content = json.loads(''.join(file), object_pairs_hook=OrderedDict) except JSONDecodeError as err: err_content = match(r'(.*): line (\d+) column (\d+)', str(err)) yield Result.from_values( self, 'This file does not contain parsable JSON. ' + err_content.group(1) + '.', file=filename, line=int(err_content.group(2)), column=int(err_content.group(3))) return corrected = json.dumps(json_content, sort_keys=json_sort, indent=indent_size, ensure_ascii=escape_unicode).splitlines(True) # Because of a bug in several python versions we have to correct # whitespace here. corrected = tuple(line.rstrip(' \n') + '\n' for line in corrected) diff = Diff.from_string_arrays(file, corrected) if len(diff) > 0 and json_sort: yield Result(self, 'This file can be reformatted by sorting keys and ' 'following indentation.', affected_code=tuple( d.range(filename) for d in diff.split_diff()), diffs={filename: diff}) elif len(diff) > 0 and not json_sort: yield Result(self, 'This file can be reformatted by ' 'following indentation.', affected_code=tuple( d.range(filename) for d in diff.split_diff()), diffs={filename: diff})
def apply(self, result, original_file_dict, file_diff_dict, editor: str): """ Open a temporary clone of the file in an editor. :param editor: The editor to open the file with. """ filename = result.file original_file = original_file_dict[filename] diff = file_diff_dict.get(filename, Diff()) current_file = diff.apply(original_file) # Prefix is nice for the user so he has an indication that its the # right file he's editing temphandle, tempname = tempfile.mkstemp(os.path.basename(filename)) os.close(temphandle) with open(tempname, "w") as temphandle: temphandle.writelines(current_file) editor_arg = EDITOR_ARGS.get(editor.strip(), None) if editor_arg: editor = editor + " " + editor_arg # Dear user, you wanted an editor, so you get it. But do you really # think you can do better than we? os.system(editor + " " + tempname) with open(tempname) as temphandle: new_file = temphandle.readlines() os.remove(tempname) intermediate_diff = Diff.from_string_arrays(current_file, new_file) file_diff_dict[filename] = diff + intermediate_diff return file_diff_dict
def get_language_tool_results(filename, file_contents, locale): joined_text = "".join(file_contents) locale = guess_language(joined_text) if locale == 'auto' else locale locale = 'en-US' if not locale else locale tool = LanguageTool(locale) matches = tool.check(joined_text) for match in matches: if not match.replacements: diffs = None else: replaced = correct(joined_text, [match]).splitlines(True) diffs = {filename: Diff.from_string_arrays(file_contents, replaced)} rule_id = match.ruleId if match.subId is not None: rule_id += '[{}]'.format(match.subId) message = match.msg + ' (' + rule_id + ')' yield message, diffs, SourceRange.from_values(filename, match.fromy+1, match.fromx+1, match.toy+1, match.tox+1)
def process_output(self, output, filename, file): if not file: return output = json.loads(output) lines = "".join(file) assert len(output) == 1 for result in output[0]['messages']: if 'fix' not in result: diffs = None else: fix = result['fix'] start, end = fix['range'] replacement_text = fix['text'] new_output = lines[:start] + replacement_text + lines[end:] diffs = {filename: Diff.from_string_arrays( lines.splitlines(True), new_output.splitlines(True))} origin = ( "{class_name} ({rule})".format(class_name=type(self).__name__, rule=result['ruleId']) if result['ruleId'] is not None else self) yield Result.from_values( origin=origin, message=result['message'], file=filename, line=result['line'], diffs=diffs, severity=self.severity_map[result['severity']])
def _get_diff(self): if self.treat_seperated_imports_independently: import_stmts = PyImportSortBear._seperate_imports(self.file) sorted_imps = [] for units in import_stmts: sort_imports = SortImports(file_contents=''.join( [x[1] for x in units]), **self.isort_settings) sort_imports = sort_imports.output.splitlines(True) sorted_imps.append((units, sort_imports)) diff = Diff(self.file) for old, new in sorted_imps: start = old[0][0] end = start + len(old) - 1 diff.delete_lines(start, end) assert isinstance(new, list) diff.add_lines(start, list(new)) if diff.modified != diff._file: return diff else: sort_imports = SortImports(file_contents=''.join(self.file), **self.isort_settings) new_file = tuple(sort_imports.output.splitlines(True)) if new_file != tuple(self.file): diff = Diff.from_string_arrays(self.file, new_file) return diff return None
def run(self, filename, file, max_line_length: int=80, tab_width: int=SpacingHelper.DEFAULT_TAB_WIDTH, pep_ignore: typed_list(str)=(), pep_select: typed_list(str)=(), local_pep8_config: bool=False): """ Detects and fixes PEP8 incompliant code. This bear will not change functionality of the code in any way. :param max_line_length: Maximum number of characters for a line. :param tab_width: Number of spaces per indent level. :param pep_ignore: A list of errors/warnings to ignore. :param pep_select: A list of errors/warnings to exclusively apply. :param local_pep8_config: Set to true if autopep8 should use a config file as if run normally from this directory. """ options = {"ignore": pep_ignore, "select": pep_select, "max_line_length": max_line_length, "indent_size": tab_width} corrected = autopep8.fix_code(''.join(file), apply_config=local_pep8_config, options=options).splitlines(True) diffs = Diff.from_string_arrays(file, corrected).split_diff() for diff in diffs: yield Result(self, "The code does not comply to PEP8.", affected_code=(diff.range(filename),), diffs={filename: diff})
def test_process_output_corrected(self): uut = (linter(sys.executable, output_format="corrected")( self.EmptyTestLinter)(self.section, None)) original = ["void main() {\n", "return 09;\n", "}\n"] fixed = ["void main()\n", "{\n", "return 9;\n", "}\n"] fixed_string = "".join(fixed) results = list( uut.process_output(fixed_string, "some-file.c", original)) diffs = list(Diff.from_string_arrays(original, fixed).split_diff()) expected = [ Result.from_values(uut, "Inconsistency found.", "some-file.c", 1, None, 2, None, RESULT_SEVERITY.NORMAL, diffs={"some-file.c": diffs[0]}) ] self.assertEqual(results, expected) # Test when providing a sequence as output. results = list( uut.process_output([fixed_string, fixed_string], "some-file.c", original)) self.assertEqual(results, 2 * expected)
def apply(self, result, original_file_dict, file_diff_dict, editor: str): """ Open the affected file(s) in an editor. :param editor: The editor to open the file with. """ # Use set to remove duplicates filenames = {src.file: src.renamed_file(file_diff_dict) for src in result.affected_code} editor_args = [editor] + list(filenames.values()) arg = EDITOR_ARGS.get(editor.strip(), None) if arg: editor_args.append(arg) # Dear user, you wanted an editor, so you get it. But do you really # think you can do better than we? if editor in GUI_EDITORS: subprocess.call(editor_args, stdout=subprocess.PIPE) else: subprocess.call(editor_args) for original_name, filename in filenames.items(): with open(filename, encoding='utf-8') as file: file_diff_dict[original_name] = Diff.from_string_arrays( original_file_dict[original_name], file.readlines(), rename=False if original_name == filename else filename) return file_diff_dict
def get_language_tool_results(filename, file_contents, locale): joined_text = "".join(file_contents) locale = guess_language(joined_text) if locale == 'auto' else locale locale = 'en-US' if not locale else locale tool = LanguageTool(locale) matches = tool.check(joined_text) for match in matches: if not match.replacements: diffs = None else: replaced = correct(joined_text, [match]).splitlines(True) diffs = { filename: Diff.from_string_arrays(file_contents, replaced) } rule_id = match.ruleId if match.subId is not None: rule_id += '[{}]'.format(match.subId) message = match.msg + ' (' + rule_id + ')' yield message, diffs, SourceRange.from_values(filename, match.fromy + 1, match.fromx + 1, match.toy + 1, match.tox + 1)
def apply(self, result, original_file_dict, file_diff_dict, editor: str): ''' Open the affected file(s) in an editor. :param editor: The editor to open the file with. ''' # Use set to remove duplicates filenames = set(src.file for src in result.affected_code) editor_args = [editor] + list(filenames) arg = EDITOR_ARGS.get(editor.strip(), None) if arg: editor_args.append(arg) # Dear user, you wanted an editor, so you get it. But do you really # think you can do better than we? if editor in GUI_EDITORS: subprocess.call(editor_args, stdout=subprocess.PIPE) else: subprocess.call(editor_args) for filename in filenames: with open(filename, encoding='utf-8') as file: new_file = file.readlines() original_file = original_file_dict[filename] try: current_file = file_diff_dict[filename].modified except KeyError: current_file = original_file file_diff_dict[filename] = Diff.from_string_arrays(original_file, new_file) return file_diff_dict
def run(self, filename, file, remove_all_unused_imports: bool=False, remove_unused_variables: bool=True): """ Detects unused code. By default this functionality is limited to: - Unneeded pass statements. - Unneeded builtin imports. :param remove_all_unused_imports: True removes all unused imports - might have side effects :param remove_unused_variables: True removes unused variables - might have side effects """ corrected = autoflake.fix_code( ''.join(file), additional_imports=None, remove_all_unused_imports=remove_all_unused_imports, remove_unused_variables=remove_unused_variables ).splitlines(True) for diff in Diff.from_string_arrays(file, corrected).split_diff(): yield Result(self, 'This file contains unused source code.', affected_code=(diff.range(filename),), diffs={filename: diff})
def process_output_corrected(self, output, filename, file, diff_severity=RESULT_SEVERITY.NORMAL, result_message='Inconsistency found.', diff_distance=1): """ Processes the executable's output as a corrected file. :param output: The output of the program as a string. :param filename: The filename of the file currently being corrected. :param file: The contents of the file currently being corrected. :param diff_severity: The severity to use for generating results. :param result_message: The message to use for generating results. :param diff_distance: Number of unchanged lines that are allowed in between two changed lines so they get yielded as one diff. If a negative distance is given, every change will be yielded as an own diff, even if they are right beneath each other. :return: An iterator returning results containing patches for the file to correct. """ return self.process_diff( Diff.from_string_arrays(file, output.splitlines(keepends=True)), filename, diff_severity, result_message, diff_distance)
def run(self, filename, file, max_line_length: int=79, indent_size: int=SpacingHelper.DEFAULT_TAB_WIDTH, pep_ignore: typed_list(str)=(), pep_select: typed_list(str)=(), local_pep8_config: bool=False): """ Detects and fixes PEP8 incompliant code. This bear will not change functionality of the code in any way. :param max_line_length: Maximum number of characters for a line. :param indent_size: Number of spaces per indentation level. :param pep_ignore: A list of errors/warnings to ignore. :param pep_select: A list of errors/warnings to exclusively apply. :param local_pep8_config: Set to true if autopep8 should use a config file as if run normally from this directory. """ options = {'ignore': pep_ignore, 'select': pep_select, 'max_line_length': max_line_length, 'indent_size': indent_size} corrected = autopep8.fix_code(''.join(file), apply_config=local_pep8_config, options=options).splitlines(True) diffs = Diff.from_string_arrays(file, corrected).split_diff() for diff in diffs: yield Result(self, 'The code does not comply to PEP8.', affected_code=(diff.range(filename),), diffs={filename: diff})
def apply(self, result, original_file_dict, file_diff_dict, editor: str): """ Open file(s) :param editor: The editor to open the file with. """ # Use set to remove duplicates filenames = { src.file: src.renamed_file(file_diff_dict) for src in result.affected_code } editor_args = [editor] + list(filenames.values()) arg = EDITOR_ARGS.get(editor.strip(), None) if arg: editor_args.append(arg) # Dear user, you wanted an editor, so you get it. But do you really # think you can do better than we? if editor in GUI_EDITORS: subprocess.call(editor_args, stdout=subprocess.PIPE) else: subprocess.call(editor_args) for original_name, filename in filenames.items(): with open(filename, encoding='utf-8') as file: file_diff_dict[original_name] = Diff.from_string_arrays( original_file_dict[original_name], file.readlines(), rename=False if original_name == filename else filename) return file_diff_dict
def filter_results(original_file_dict, modified_file_dict, original_results, modified_results): """ Filters results for such ones that are unique across file changes :param original_file_dict: Dict of lists of file contents before changes :param modified_file_dict: Dict of lists of file contents after changes :param original_results: List of results of the old files :param modified_results: List of results of the new files :return: List of results from new files that are unique from all those that existed in the old changes """ # diffs_dict[file] is a diff between the original and modified file diffs_dict = {} for file in original_file_dict: diffs_dict[file] = Diff.from_string_arrays(original_file_dict[file], modified_file_dict[file]) orig_result_diff_dict_dict = remove_result_ranges_diffs(original_results, original_file_dict) mod_result_diff_dict_dict = remove_result_ranges_diffs(modified_results, modified_file_dict) for m_r in modified_results: for o_r in original_results: if basics_match(o_r, m_r): if source_ranges_match( original_file_dict, diffs_dict, orig_result_diff_dict_dict[o_r], mod_result_diff_dict_dict[m_r] ): # at least one original result matches completely modified_results.remove(m_r) break # only those ones left that have no perfect match return modified_results
def process_output(self, output, filename, file): if not file: return output = json.loads(output) lines = "".join(file) assert len(output) == 1 for result in output[0]['messages']: if 'fix' not in result: diffs = None else: fix = result['fix'] start, end = fix['range'] replacement_text = fix['text'] new_output = lines[:start] + replacement_text + lines[end:] diffs = { filename: Diff.from_string_arrays(lines.splitlines(True), new_output.splitlines(True)) } origin = ("{class_name} ({rule})".format( class_name=type(self).__name__, rule=result['ruleId']) if result['ruleId'] is not None else self) yield Result.from_values( origin=origin, message=result['message'], file=filename, line=result['line'], diffs=diffs, severity=self.severity_map[result['severity']])
def apply(self, result, original_file_dict, file_diff_dict, editor: str): ''' Open the affected file(s) in an editor. :param editor: The editor to open the file with. ''' # Use set to remove duplicates filenames = set(src.file for src in result.affected_code) editor_args = [editor] + list(filenames) arg = EDITOR_ARGS.get(editor.strip(), None) if arg: editor_args.append(arg) # Dear user, you wanted an editor, so you get it. But do you really # think you can do better than we? if editor in GUI_EDITORS: subprocess.call(editor_args, stdout=subprocess.PIPE) else: subprocess.call(editor_args) for filename in filenames: with open(filename, encoding='utf-8') as file: new_file = file.readlines() original_file = original_file_dict[filename] try: current_file = file_diff_dict[filename].modified except KeyError: current_file = original_file file_diff_dict[filename] = Diff.from_string_arrays( original_file, new_file) return file_diff_dict
def _get_diff(self): if self.treat_seperated_imports_independently: import_stmts = PyImportSortBear._seperate_imports(self.file) sorted_imps = [] for units in import_stmts: sort_imports = SortImports(file_contents=''. join([x[1] for x in units]), **self.isort_settings) sort_imports = sort_imports.output.splitlines(True) sorted_imps.append((units, sort_imports)) diff = Diff(self.file) for old, new in sorted_imps: start = old[0][0] end = start + len(old) - 1 diff.delete_lines(start, end) assert isinstance(new, list) diff.add_lines(start, list(new)) if diff.modified != diff._file: return diff else: sort_imports = SortImports(file_contents=''.join(self.file), **self.isort_settings) new_file = tuple(sort_imports.output.splitlines(True)) if new_file != tuple(self.file): diff = Diff.from_string_arrays(self.file, new_file) return diff return None
def run(self, filename, file, remove_all_unused_imports: bool = False, remove_unused_variables: bool = True): """ Detects unused code. By default this functionality is limited to: - Unneeded pass statements. - Unneeded builtin imports. :param remove_all_unused_imports: True removes all unused imports - might have side effects :param remove_unused_variables: True removes unused variables - might have side effects """ corrected = autoflake.fix_code( ''.join(file), additional_imports=None, remove_all_unused_imports=remove_all_unused_imports, remove_unused_variables=remove_unused_variables).splitlines(True) for diff in Diff.from_string_arrays(file, corrected).split_diff(): yield Result(self, 'This file contains unused source code.', affected_code=(diff.range(filename), ), diffs={filename: diff})
def test_stdin_stderr_config_correction(self): create_arguments_mock = Mock() generate_config_mock = Mock() # `some_value_A` and `some_value_B` are used to test the different # delegation to `generate_config()` and `create_arguments()` # accordingly. class Handler: @staticmethod def generate_config(filename, file, some_value_A): generate_config_mock(filename, file, some_value_A) return "\n".join(["use_stdin", "use_stderr", "correct"]) @staticmethod def create_arguments(filename, file, config_file, some_value_B): create_arguments_mock(filename, file, config_file, some_value_B) return self.test_program_path, "--config", config_file uut = (linter(sys.executable, use_stdin=True, use_stdout=False, use_stderr=True, output_format="corrected", config_suffix=".conf") (Handler) (self.section, None)) results = list(uut.run(self.testfile2_path, self.testfile2_content, some_value_A=124, some_value_B=-78)) expected_correction = [s + "\n" for s in ["+", "/", "/", "-"]] diffs = list(Diff.from_string_arrays( self.testfile2_content, expected_correction).split_diff()) expected = [Result.from_values(uut, "Inconsistency found.", self.testfile2_path, 1, None, 1, None, RESULT_SEVERITY.NORMAL, diffs={self.testfile2_path: diffs[0]}), Result.from_values(uut, "Inconsistency found.", self.testfile2_path, 5, None, 5, None, RESULT_SEVERITY.NORMAL, diffs={self.testfile2_path: diffs[1]})] self.assertEqual(results, expected) create_arguments_mock.assert_called_once_with( self.testfile2_path, self.testfile2_content, ANY, -78) self.assertEqual(create_arguments_mock.call_args[0][2][-5:], ".conf") generate_config_mock.assert_called_once_with( self.testfile2_path, self.testfile2_content, 124)
def test_nostdin_nostderr_noconfig_correction(self): create_arguments_mock = Mock() class Handler: @staticmethod def create_arguments(filename, file, config_file): create_arguments_mock(filename, file, config_file) return self.test_program_path, "--correct", filename uut = (linter(sys.executable, output_format="corrected", diff_severity=RESULT_SEVERITY.INFO, diff_message="Custom message")(Handler)(self.section, None)) results = list(uut.run(self.testfile_path, self.testfile_content)) expected_correction = [ s + "\n" for s in ["+", "-", "*", "++", "-", "-", "+"] ] diffs = list( Diff.from_string_arrays(self.testfile_content, expected_correction).split_diff()) expected = [ Result.from_values(uut, "Custom message", self.testfile_path, 4, None, 4, None, RESULT_SEVERITY.INFO, diffs={self.testfile_path: diffs[0]}), Result.from_values(uut, "Custom message", self.testfile_path, 6, None, 6, None, RESULT_SEVERITY.INFO, diffs={self.testfile_path: diffs[1]}), Result.from_values(uut, "Custom message", self.testfile_path, 10, None, 10, None, RESULT_SEVERITY.INFO, diffs={self.testfile_path: diffs[2]}) ] self.assertEqual(results, expected) create_arguments_mock.assert_called_once_with(self.testfile_path, self.testfile_content, None)
def run(self, filename, file, natural_language: str = 'auto', languagetool_disable_rules: typed_list(str) = (), ): """ Checks the code with LanguageTool. :param natural_language: A locale representing the language you want to have checked. If set to 'auto' the language is guessed. If the language cannot be guessed or an unsupported language is guessed, 'en-US' is used. :param languagetool_disable_rules: List of rules to disable checks for. """ # Defer import so the check_prerequisites can be run without # language_check being there. from language_check import LanguageTool, correct joined_text = ''.join(file) natural_language = (guess_language(joined_text) if natural_language == 'auto' else natural_language) try: tool = LanguageTool(natural_language, motherTongue='en_US') except ValueError: # Using 'en-US' if guessed language is not supported logging.warn( "Changing the `natural_language` setting to 'en-US' as " '`language_check` failed to guess a valid language.' ) natural_language = 'en-US' tool = LanguageTool(natural_language, motherTongue='en_US') tool.disabled.update(languagetool_disable_rules) matches = tool.check(joined_text) for match in matches: if not match.replacements: diffs = None else: replaced = correct(joined_text, [match]).splitlines(True) diffs = {filename: Diff.from_string_arrays(file, replaced)} rule_id = match.ruleId if match.subId is not None: rule_id += '[{}]'.format(match.subId) message = match.msg + ' (' + rule_id + ')' source_range = SourceRange.from_values(filename, match.fromy+1, match.fromx+1, match.toy+1, match.tox+1) yield Result(self, message, diffs=diffs, affected_code=(source_range,))
def test_stdin_stderr_config_correction(self): create_arguments_mock = Mock() generate_config_mock = Mock() # `some_value_A` and `some_value_B` are used to test the different # delegation to `generate_config()` and `create_arguments()` # accordingly. class Handler: @staticmethod def generate_config(filename, file, some_value_A): generate_config_mock(filename, file, some_value_A) return "\n".join(["use_stdin", "use_stderr", "correct"]) @staticmethod def create_arguments(filename, file, config_file, some_value_B): create_arguments_mock(filename, file, config_file, some_value_B) return self.test_program_path, "--config", config_file uut = (linter(sys.executable, use_stdin=True, use_stdout=False, use_stderr=True, output_format="corrected", config_suffix=".conf") (Handler) (self.section, None)) results = list(uut.run(self.testfile2_path, self.testfile2_content, some_value_A=124, some_value_B=-78)) expected_correction = [s + "\n" for s in ["+", "/", "/", "-"]] diffs = list(Diff.from_string_arrays( self.testfile2_content, expected_correction).split_diff()) expected = [Result.from_values(uut, "Inconsistency found.", self.testfile2_path, 1, None, 1, None, RESULT_SEVERITY.NORMAL, diffs={self.testfile2_path: diffs[0]}), Result.from_values(uut, "Inconsistency found.", self.testfile2_path, 5, None, 5, None, RESULT_SEVERITY.NORMAL, diffs={self.testfile2_path: diffs[1]})] self.assertEqual(results, expected) create_arguments_mock.assert_called_once_with( self.testfile2_path, self.testfile2_content, ANY, -78) self.assertEqual(create_arguments_mock.call_args[0][2][-5:], ".conf") generate_config_mock.assert_called_once_with( self.testfile2_path, self.testfile2_content, 124)
def test_json_export(self): a = ["first\n", "second\n", "third\n"] b = ["first\n", "third\n"] diff = Diff.from_string_arrays(a, b) self.assertEqual( json.dumps(diff, cls=JSONEncoder, sort_keys=True), '"--- \\n' "+++ \\n" "@@ -1,3 +1,2 @@\\n" " first\\n" "-second\\n" ' third\\n"', )
def run(self, filename, file, natural_language: str='auto', languagetool_disable_rules: typed_list(str)=()): ''' Checks the code with LanguageTool. :param natural_language: A locale representing the language you want to have checked. If set to 'auto' the language is guessed. If the language cannot be guessed or an unsupported language is guessed, 'en-US' is used. :param languagetool_disable_rules: List of rules to disable checks for. ''' # Defer import so the check_prerequisites can be run without # language_check being there. from language_check import LanguageTool, correct joined_text = ''.join(file) natural_language = (guess_language(joined_text) if natural_language == 'auto' else natural_language) try: tool = LanguageTool(natural_language, motherTongue='en_US') except ValueError: # Using 'en-US' if guessed language is not supported logging.warn( "Changing the `natural_language` setting to 'en-US' as " '`language_check` failed to guess a valid language.' ) natural_language = 'en-US' tool = LanguageTool(natural_language, motherTongue='en_US') tool.disabled.update(languagetool_disable_rules) matches = tool.check(joined_text) for match in matches: if not match.replacements: diffs = None else: replaced = correct(joined_text, [match]).splitlines(True) diffs = {filename: Diff.from_string_arrays(file, replaced)} rule_id = match.ruleId if match.subId is not None: rule_id += '[{}]'.format(match.subId) message = match.msg + ' (' + rule_id + ')' source_range = SourceRange.from_values(filename, match.fromy+1, match.fromx+1, match.toy+1, match.tox+1) yield Result(self, message, diffs=diffs, affected_code=(source_range,))
def run( self, filename, file, max_line_length: int = 79, indent_size: int = SpacingHelper.DEFAULT_TAB_WIDTH, pep_ignore: typed_list(str) = (), pep_select: typed_list(str) = (), local_pep8_config: bool = False, ): """ Detects and fixes PEP8 incompliant code in Jupyter Notebooks. This bear will not change functionality of the code in any way. :param max_line_length: Maximum number of characters for a line. When set to 0 allows infinite line length. :param indent_size: Number of spaces per indent level. :param pep_ignore: A list of errors/warnings to ignore. :param pep_select: A list of errors/warnings to exclusively apply. :param local_pep8_config: Set to true if autopep8 should use a config file as if run normally from this directory. """ if not max_line_length: max_line_length = sys.maxsize options = { 'ignore': pep_ignore, 'select': pep_select, 'max_line_length': max_line_length, 'indent_size': indent_size } notebook_node = notebook_node_from_string_list(file) cells = notebook_node['cells'] for cell in cells: if cell['cell_type'] != 'code': continue cell['source'] = autopep8_fix_code_cell(cell['source'], local_pep8_config, options) corrected = notebook_node_to_string_list(notebook_node) # If newline at eof in `file` but not in `corrected`, add # final newline character to `corrected` to make sure this difference # does not pop up in `diffs`. if file[-1].endswith('\n') and not corrected[-1].endswith('\n'): corrected[-1] += '\n' diffs = Diff.from_string_arrays(file, corrected).split_diff() for diff in diffs: yield Result(self, 'The code does not comply to PEP8.', affected_code=(diff.range(filename), ), diffs={filename: diff})
def test_equality(self): a = ['first', 'second', 'third'] b = ['first', 'third'] diff_1 = Diff.from_string_arrays(a, b) a[1] = 'else' diff_2 = Diff.from_string_arrays(a, b) self.assertEqual(diff_1, diff_2) diff_1.rename = 'abcd' self.assertNotEqual(diff_1, diff_2) diff_1.rename = False diff_1.delete = True self.assertNotEqual(diff_1, diff_2) diff_1.delete = False diff_1.add_lines(1, ['1']) self.assertNotEqual(diff_1, diff_2)
def test_equality(self): a = ['first', 'second', 'third'] b = ['first', 'third'] diff_1 = Diff.from_string_arrays(a, b) a[1] = 'else' diff_2 = Diff.from_string_arrays(a, b) self.assertEqual(diff_1, diff_2) diff_1.rename = 'abcd' self.assertNotEqual(diff_1, diff_2) diff_1.rename = False diff_1.delete = True self.assertNotEqual(diff_1, diff_2) diff_1.delete = False diff_1.add_lines(1, ['1']) self.assertNotEqual(diff_1, diff_2)
def run(self, filename, file): """ Detects commented out source code in Python. """ corrected = tuple(eradicate.filter_commented_out_code(''.join(file))) for diff in Diff.from_string_arrays(file, corrected).split_diff(): yield Result(self, "This file contains commented out source code.", affected_code=(diff.range(filename),), diffs={filename: diff})
def test_json_export(self): a = ["first\n", "second\n", "third\n"] b = ["first\n", "third\n"] diff = Diff.from_string_arrays(a, b) self.assertEqual( json.dumps(diff, cls=JSONEncoder, sort_keys=True), '"--- \\n' '+++ \\n' '@@ -1,3 +1,2 @@\\n' ' first\\n' '-second\\n' ' third\\n"')
def test_no_range(self): test_file = ['abc'] test_file_dict = {abspath('test_file'): test_file} test_result = Result('origin', 'message') result_diff = remove_result_ranges_diffs( [test_result], test_file_dict)[test_result][abspath('test_file')] expected_diff = Diff.from_string_arrays(test_file, ['abc']) self.assertEqual(result_diff, expected_diff)
def test_no_range(self): test_file = ["abc"] test_file_dict = {abspath("test_file"): test_file} test_result = Result("origin", "message") result_diff = remove_result_ranges_diffs( [test_result], test_file_dict)[test_result][abspath("test_file")] expected_diff = Diff.from_string_arrays(test_file, ["abc"]) self.assertEqual(result_diff, expected_diff)
def run(self, filename, file): """ Detects commented out source code in Python. """ corrected = tuple(eradicate.filter_commented_out_code(''.join(file))) for diff in Diff.from_string_arrays(file, corrected).split_diff(): yield Result(self, 'This file contains commented out source code.', affected_code=(diff.range(filename),), diffs={filename: diff})
def apply(self, result, original_file_dict, file_diff_dict, editor: str): """ (O)pen file :param editor: The editor to open the file with. """ try: editor_info = KNOWN_EDITORS[editor.strip()] except KeyError: # If the editor is unknown fall back to just passing # the filenames and emit a warning logging.warning( 'The editor "{editor}" is unknown to coala. Files won\'t be' ' opened at the correct positions and other quirks might' ' occur. Consider opening an issue at' ' https://github.com/coala/coala/issues so we' ' can add support for this editor.' ' Supported editors are: {supported}'.format( editor=editor, supported=', '.join( sorted(KNOWN_EDITORS.keys()) ) ) ) editor_info = { 'file_arg_template': '{filename}', 'gui': False } # Use dict to remove duplicates filenames = { src.file: { 'filename': src.renamed_file(file_diff_dict), 'line': src.start.line or 1, 'column': src.start.column or 1 } for src in result.affected_code } call_args = self.build_editor_call_args(editor, editor_info, filenames) if editor_info.get('gui', True): subprocess.call(call_args, stdout=subprocess.PIPE) else: subprocess.call(call_args) for original_name, file_info in filenames.items(): filename = file_info['filename'] with open(filename, encoding=detect_encoding(filename)) as file: file_diff_dict[original_name] = Diff.from_string_arrays( original_file_dict[original_name], file.readlines(), rename=False if original_name == filename else filename) return file_diff_dict
def run(self, filename, file): """ Sorts imports for python. """ new_file = SortImports( file_contents=''.join(file)).output.splitlines(True) if new_file != file: diff = Diff.from_string_arrays(file, new_file) yield Result(self, "Imports can be sorted.", affected_code=diff.affected_code(filename), diffs={filename: diff})
def remove_result_ranges_diffs(result_list, file_dict): """ Calculates the diffs to all files in file_dict that describe the removal of each respective result's affected code. :param result_list: list of results :param file_dict: dict of file contents :return: returnvalue[result][file] is a diff of the changes the removal of this result's affected code would cause for the file. """ result_diff_dict_dict = {} for original_result in result_list: mod_file_dict = copy.deepcopy(file_dict) # gather all source ranges from this result source_ranges = [] # SourceRanges must be sorted backwards and overlaps must be eliminated # this way, the deletion based on sourceRanges is not offset by # previous deletions in the same line that invalidate the indices. previous = None for source_range in sorted(original_result.affected_code, reverse=True): # previous exists and overlaps if previous is not None and source_range.overlaps(previous): combined_sr = SourceRange.join(previous, source_range) previous = combined_sr elif previous is None: previous = source_range # previous exists but it doesn't overlap else: source_ranges.append(previous) previous = source_range # don't forget last entry if there were any: if previous: source_ranges.append(previous) for source_range in source_ranges: file_name = source_range.file new_file = remove_range(mod_file_dict[file_name], source_range) mod_file_dict[file_name] = new_file diff_dict = {} for file_name in file_dict: diff_dict[file_name] = Diff.from_string_arrays( file_dict[file_name], mod_file_dict[file_name]) result_diff_dict_dict[original_result] = diff_dict return result_diff_dict_dict
def remove_result_ranges_diffs(result_list, file_dict): """ Calculates the diffs to all files in file_dict that describe the removal of each respective result's affected code. :param result_list: list of results :param file_dict: dict of file contents :return: returnvalue[result][file] is a diff of the changes the removal of this result's affected code would cause for the file. """ result_diff_dict_dict = {} for original_result in result_list: mod_file_dict = copy.deepcopy(file_dict) # gather all source ranges from this result source_ranges = [] # SourceRanges must be sorted backwards and overlaps must be eliminated # this way, the deletion based on sourceRanges is not offset by # previous deletions in the same line that invalidate the indices. previous = None for source_range in sorted(original_result.affected_code, reverse=True): # previous exists and overlaps if previous is not None and source_range.overlaps(previous): combined_sr = SourceRange.join(previous, source_range) previous = combined_sr elif previous is None: previous = source_range # previous exists but it doesn't overlap else: source_ranges.append(previous) previous = source_range # don't forget last entry if there were any: if previous: source_ranges.append(previous) for source_range in source_ranges: file_name = source_range.file new_file = remove_range(mod_file_dict[file_name], source_range) mod_file_dict[file_name] = new_file diff_dict = {} for file_name in file_dict: diff_dict[file_name] = Diff.from_string_arrays( file_dict[file_name], mod_file_dict[file_name]) result_diff_dict_dict[original_result] = diff_dict return result_diff_dict_dict
def run(self, filename, file, max_line_length: int = 79, indent_size: int = SpacingHelper.DEFAULT_TAB_WIDTH, pep_ignore: typed_list(str) = (), pep_select: typed_list(str) = (), local_pep8_config: bool = False, ): """ Detects and fixes PEP8 incompliant code in Jupyter Notebooks. This bear will not change functionality of the code in any way. :param max_line_length: Maximum number of characters for a line. When set to 0 allows infinite line length. :param indent_size: Number of spaces per indent level. :param pep_ignore: A list of errors/warnings to ignore. :param pep_select: A list of errors/warnings to exclusively apply. :param local_pep8_config: Set to true if autopep8 should use a config file as if run normally from this directory. """ if not max_line_length: max_line_length = sys.maxsize options = {'ignore': pep_ignore, 'select': pep_select, 'max_line_length': max_line_length, 'indent_size': indent_size} notebook_node = notebook_node_from_string_list(file) cells = notebook_node['cells'] for cell in cells: if cell['cell_type'] != 'code': continue cell['source'] = autopep8_fix_code_cell(cell['source'], local_pep8_config, options) corrected = notebook_node_to_string_list(notebook_node) # If newline at eof in `file` but not in `corrected`, add # final newline character to `corrected` to make sure this difference # does not pop up in `diffs`. if file[-1].endswith('\n') and not corrected[-1].endswith('\n'): corrected[-1] += '\n' diffs = Diff.from_string_arrays(file, corrected).split_diff() for diff in diffs: yield Result(self, 'The code does not comply to PEP8.', affected_code=(diff.range(filename),), diffs={filename: diff})
def process_output(self, output, filename, file): def warn_issue(message): self.warn('While running {0}, some issues were found:'.format( self.__class__.__name__)) self.warn(message) # Taking output from stderr for ESLint 2 program errors. # ESLint 2 is no longer supported, but it is here so that anyone # with ESLint 2 will still see any errors that were emitted. if output[1]: # pragma: no cover warn_issue(output[1]) if not file or not output[0]: return # Handling program errors # such as malformed config file or missing plugin # that are not in json format. try: output = json.loads(output[0]) except ValueError: warn_issue(output[0]) return lines = ''.join(file) assert len(output) == 1 for result in output[0]['messages']: if 'fix' not in result: diffs = None else: fix = result['fix'] start, end = fix['range'] replacement_text = fix['text'] new_output = lines[:start] + replacement_text + lines[end:] diffs = { filename: Diff.from_string_arrays(lines.splitlines(True), new_output.splitlines(True)) } origin = ('{class_name} ({rule})'.format( class_name=type(self).__name__, rule=result['ruleId']) if result.get('ruleId') is not None else self) yield Result.from_values( origin=origin, message=result['message'], file=filename, line=result.get('line'), diffs=diffs, severity=self.severity_map[result['severity']])
def test_json_export(self): JSONEncoder = create_json_encoder() a = ['first\n', 'second\n', 'third\n'] b = ['first\n', 'third\n'] diff = Diff.from_string_arrays(a, b) self.assertEqual( json.dumps(diff, cls=JSONEncoder, sort_keys=True), '"--- \\n' '+++ \\n' '@@ -1,3 +1,2 @@\\n' ' first\\n' '-second\\n' ' third\\n"')
def test_no_range(self): test_file = ['abc'] test_file_dict = {abspath('test_file'): test_file} test_result = Result('origin', 'message') result_diff = remove_result_ranges_diffs( [test_result], test_file_dict)[test_result][abspath('test_file')] expected_diff = Diff.from_string_arrays(test_file, ['abc']) self.assertEqual(result_diff, expected_diff)
def test_no_range(self): test_file = ["abc"] test_file_dict = {abspath("test_file"): test_file} test_result = Result("origin", "message") result_diff = remove_result_ranges_diffs( [test_result], test_file_dict)[test_result][abspath("test_file")] expected_diff = Diff.from_string_arrays(test_file, ["abc"]) self.assertEqual(result_diff, expected_diff)
def filter_results(original_file_dict, modified_file_dict, original_results, modified_results): """ Filters results for such ones that are unique across file changes :param original_file_dict: Dict of lists of file contents before changes :param modified_file_dict: Dict of lists of file contents after changes :param original_results: List of results of the old files :param modified_results: List of results of the new files :return: List of results from new files that are unique from all those that existed in the old changes """ renamed_files = ensure_files_present(original_file_dict, modified_file_dict) # diffs_dict[file] is a diff between the original and modified file diffs_dict = {} for file in original_file_dict: diffs_dict[file] = Diff.from_string_arrays( original_file_dict[file], modified_file_dict[renamed_files.get(file, file)]) orig_result_diff_dict_dict = remove_result_ranges_diffs(original_results, original_file_dict) mod_result_diff_dict_dict = remove_result_ranges_diffs(modified_results, modified_file_dict) unique_results = [] for m_r in reversed(modified_results): unique = True for o_r in original_results: if basics_match(o_r, m_r): if source_ranges_match(original_file_dict, diffs_dict, orig_result_diff_dict_dict[o_r], mod_result_diff_dict_dict[m_r], renamed_files): # at least one original result matches completely unique = False break if unique: unique_results.append(m_r) return unique_results
def run(self, filename, file, json_sort: bool=False, indent_size: int=SpacingHelper.DEFAULT_TAB_WIDTH, escape_unicode: bool=True): """ Raises issues for any deviations from the pretty-printed JSON. :param json_sort: Whether or not keys should be sorted. :param indent_size: Number of spaces per indentation level. :param escape_unicode: Whether or not to escape unicode values using ASCII. """ # Output a meaningful message if empty file given as input if len(file) == 0: yield Result.from_values(self, 'This file is empty.', file=filename) return try: json_content = json.loads(''.join(file), object_pairs_hook=OrderedDict) except JSONDecodeError as err: err_content = match(r'(.*): line (\d+) column (\d+)', str(err)) yield Result.from_values( self, 'This file does not contain parsable JSON. ' + err_content.group(1) + '.', file=filename, line=int(err_content.group(2)), column=int(err_content.group(3))) return corrected = json.dumps(json_content, sort_keys=json_sort, indent=indent_size, ensure_ascii=escape_unicode ).splitlines(True) # Because of a bug in several python versions we have to correct # whitespace here. corrected = tuple(line.rstrip(' \n') + '\n' for line in corrected) diff = Diff.from_string_arrays(file, corrected) if len(diff) > 0: yield Result(self, 'This file can be reformatted by sorting keys and ' 'following indentation.', affected_code=tuple(d.range(filename) for d in diff.split_diff()), diffs={filename: diff})