def test_print_results_multiple_ranges(self): affected_code = (SourceRange.from_values('some_file', 5, end_line=7), SourceRange.from_values('another_file', 1, 3, 1, 5), SourceRange.from_values('another_file', 3, 3, 3, 5)) with retrieve_stdout() as stdout: print_results( self.log_printer, Section(''), [ Result('ClangCloneDetectionBear', 'Clone Found', affected_code) ], { abspath('some_file'): ['line ' + str(i + 1) + '\n' for i in range(10)], abspath('another_file'): ['line ' + str(i + 1) for i in range(10)] }, {}, self.console_printer) self.assertEqual( """li{0}{1} li{0}{2} **** ClangCloneDetectionBear [Section: ] **** ! ! [Severity: NORMAL] ! ! {3}\n""".format( highlight_text(self.no_color, 'ne', self.lexer, BackgroundSourceRangeStyle), highlight_text(self.no_color, ' 1', self.lexer), highlight_text(self.no_color, ' 3', self.lexer), highlight_text(self.no_color, 'Clone Found', style=BackgroundMessageStyle)), stdout.getvalue())
def test_settings(self): uut = (external_bear_wrap(sys.executable, settings={ 'set_normal_severity': ('', bool), 'set_sample_dbg_msg': ('', bool, False), 'not_set_different_msg': ('', bool, True)}) (self.TestBear) (self.section, None)) results = list(uut.run(self.testfile_path, self.testfile_content, set_normal_severity=False, set_sample_dbg_msg=True, not_set_different_msg=False)) expected = [ Result( origin=uut, message='This is wrong', affected_code=(SourceRange.from_values(self.testfile_path, 1),), severity=RESULT_SEVERITY.MAJOR, debug_msg='Sample debug message' ), Result( origin=uut, message='Different message', affected_code=(SourceRange.from_values(self.testfile_path, 3),), severity=RESULT_SEVERITY.INFO)] self.assertEqual(results, expected)
def range(self, filename): """ Calculates a SourceRange spanning over the whole Diff. If something is added after the 0th line (i.e. before the first line) the first line will be included in the SourceRange. The range of an empty diff will only affect the filename: >>> range = Diff([]).range("file") >>> range.file is None False >>> print(range.start.line) None :param filename: The filename to associate the SourceRange with. :return: A SourceRange object. """ if len(self._changes) == 0: return SourceRange.from_values(filename) start = min(self._changes.keys()) end = max(self._changes.keys()) return SourceRange.from_values(filename, start_line=max(1, start), end_line=max(1, end))
def yield_ignore_ranges(file_dict): """ Yields tuples of affected bears and a SourceRange that shall be ignored for those. :param file_dict: The file dictionary. """ for filename, file in file_dict.items(): start = None bears = [] for line_number, line in enumerate(file, start=1): line = line.lower() if "start ignoring " in line: start = line_number bears = get_ignore_scope(line, "start ignoring ") elif "stop ignoring" in line: if start: yield (bears, SourceRange.from_values(filename, start, end_line=line_number)) elif "ignore " in line: yield (get_ignore_scope(line, "ignore "), SourceRange.from_values(filename, line_number, end_line=line_number + 1))
def test_ignore_glob(self): result = Result.from_values("LineLengthBear", "message", file="d", line=1, column=1, end_line=2, end_column=2) ranges = [(["(line*|space*)", "py*"], SourceRange.from_values("d", 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges)) result = Result.from_values("SpaceConsistencyBear", "message", file="d", line=1, column=1, end_line=2, end_column=2) ranges = [(["(line*|space*)", "py*"], SourceRange.from_values("d", 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges)) result = Result.from_values("XMLBear", "message", file="d", line=1, column=1, end_line=2, end_column=2) ranges = [(["(line*|space*)", "py*"], SourceRange.from_values("d", 1, 1, 2, 2))] self.assertFalse(check_result_ignore(result, ranges))
def test_settings(self): uut = (external_bear_wrap(sys.executable, settings={ 'set_normal_severity': ('', bool), 'set_sample_dbg_msg': ('', bool, False), 'not_set_different_msg': ('', bool, True) })(self.TestBear)(self.section, None)) results = list( uut.run(self.testfile_path, self.testfile_content, set_normal_severity=False, set_sample_dbg_msg=True, not_set_different_msg=False)) expected = [ Result(origin=uut, message='This is wrong', affected_code=(SourceRange.from_values( self.testfile_path, 1), ), severity=RESULT_SEVERITY.MAJOR, debug_msg='Sample debug message'), Result(origin=uut, message='Different message', affected_code=(SourceRange.from_values( self.testfile_path, 3), ), severity=RESULT_SEVERITY.INFO) ] self.assertEqual(results, expected)
def test_ignore_results(self): ranges = [([], SourceRange.from_values("f", 1, 1, 2, 2))] result = Result.from_values("origin", "message", file="e", line=1, column=1, end_line=2, end_column=2) self.assertFalse(check_result_ignore(result, ranges)) ranges.append(([], SourceRange.from_values("e", 2, 3, 3, 3))) self.assertFalse(check_result_ignore(result, ranges)) ranges.append(([], SourceRange.from_values("e", 1, 1, 2, 2))) self.assertTrue(check_result_ignore(result, ranges)) result1 = Result.from_values("origin", "message", file="e") self.assertFalse(check_result_ignore(result1, ranges)) ranges = [(['something', 'else', 'not origin'], SourceRange.from_values("e", 1, 1, 2, 2))] self.assertFalse(check_result_ignore(result, ranges)) ranges = [(['something', 'else', 'origin'], SourceRange.from_values("e", 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges))
def test_print_results_multiple_ranges(self): affected_code = ( SourceRange.from_values("some_file", 5, end_line=7), SourceRange.from_values("another_file", 1, 3, 1, 5), SourceRange.from_values("another_file", 3, 3, 3, 5)) with retrieve_stdout() as stdout: print_results( self.log_printer, Section(""), [Result("ClangCloneDetectionBear", "Clone Found", affected_code)], {abspath("some_file"): ["line " + str(i + 1) + "\n" for i in range(10)], abspath("another_file"): ["line " + str(i + 1) + "\n" for i in range(10)]}, {}, color=False) self.assertEqual(""" another_file | 1| line•1 another_file | 3| line•3 some_file | 5| line•5 | 6| line•6 | 7| line•7 | | [NORMAL] ClangCloneDetectionBear: | | Clone Found """, stdout.getvalue())
def test_keyword_diff(self): text = ['# todo 123\n'] comments = [SourceRange.from_values('F', 1, 1, 1, 10)] dep_results = { 'AnnotationBear': [ self.annotation_bear_result_type({'comments': comments}) ] } with execute_bear(self.uut, filename='F', file=text, dependency_results=dep_results) as result: self.assertEqual(result[0].diffs['F'].unified_diff, '--- \n' '+++ \n' '@@ -1 +0,0 @@\n' '-# todo 123\n') text = ['test = 55 # todo 123\n'] comments = [SourceRange.from_values('F', 1, 11, 1, 23)] dep_results = { 'AnnotationBear': [ self.annotation_bear_result_type({'comments': comments}) ] } with execute_bear(self.uut, filename='F', file=text, dependency_results=dep_results) as result: self.assertEqual(result[0].diffs['F'].unified_diff, '--- \n' '+++ \n' '@@ -1 +1 @@\n' '-test = 55 # todo 123\n' '+test = 55\n')
def range(self, filename): """ Calculates a SourceRange spanning over the whole Diff. If something is added after the 0th line (i.e. before the first line) the first line will be included in the SourceRange. The range of an empty diff will only affect the filename: >>> range = Diff([]).range("file") >>> range.file is None False >>> print(range.start.line) None :param filename: The filename to associate the SourceRange with. :return: A SourceRange object. """ if len(self._changes) == 0: return SourceRange.from_values(filename) start = min(self._changes.keys()) end = max(self._changes.keys()) return SourceRange.from_values(filename, start_line=max(1, start), end_line=max(1, end))
def test_overlaps(self): a = SourceRange.from_values('test_file', 2, None, 3) b = SourceRange.from_values('test_file', 3, None, 5) self.assertTrue(a.overlaps(b)) self.assertTrue(b.overlaps(a)) a = SourceRange.from_values('test_file1', 2, None, 3) b = SourceRange.from_values('test_file2', 3, None, 5) self.assertFalse(a.overlaps(b)) self.assertFalse(b.overlaps(a)) a = SourceRange.from_values('test_file', 2, None, 2, None) b = SourceRange.from_values('test_file', 2, 2, 2, 80) self.assertTrue(a.overlaps(b)) self.assertTrue(b.overlaps(a)) a = SourceRange.from_values('test_file1', 1, None, None, None) b = SourceRange.from_values('test_file2', 1, None, 1, None) self.assertFalse(a.overlaps(b)) self.assertFalse(b.overlaps(a)) a = SourceRange.from_values('test_file', 1, None, None, None) b = SourceRange.from_values('test_file', 1, None, 1, None) self.assertTrue(a.overlaps(b)) self.assertTrue(b.overlaps(a))
def yield_ignore_ranges(file_dict): """ Yields tuples of affected bears and a SourceRange that shall be ignored for those. :param file_dict: The file dictionary. """ for filename, file in file_dict.items(): start = None bears = [] stop_ignoring = False for line_number, line in enumerate(file, start=1): # Before lowering all lines ever read, first look for the biggest # common substring, case sensitive: I*gnor*e, start i*gnor*ing. if 'gnor' in line: line = line.lower() if "start ignoring " in line: start = line_number bears = get_ignore_scope(line, "start ignoring ") elif "stop ignoring" in line: stop_ignoring = True if start: yield (bears, SourceRange.from_values( filename, start, 1, line_number, len(file[line_number - 1]))) elif "ignore " in line: yield (get_ignore_scope(line, "ignore "), SourceRange.from_values(filename, line_number, 1, line_number + 1, len(file[line_number]))) if stop_ignoring is False and start is not None: yield (bears, SourceRange.from_values(filename, start, 1, len(file), len(file[-1])))
def test_ignore_glob(self): result = Result.from_values('LineLengthBear', 'message', file='d', line=1, column=1, end_line=2, end_column=2) ranges = [(['(line*|space*)', 'py*'], SourceRange.from_values('d', 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges)) result = Result.from_values('SpaceConsistencyBear', 'message', file='d', line=1, column=1, end_line=2, end_column=2) ranges = [(['(line*|space*)', 'py*'], SourceRange.from_values('d', 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges)) result = Result.from_values('XMLBear', 'message', file='d', line=1, column=1, end_line=2, end_column=2) ranges = [(['(line*|space*)', 'py*'], SourceRange.from_values('d', 1, 1, 2, 2))] self.assertFalse(check_result_ignore(result, ranges))
def test_ignore_results(self): ranges = [([], SourceRange.from_values("f", 1, 1, 2, 2))] result = Result.from_values("origin", "message", file="e", line=1, column=1, end_line=2, end_column=2) self.assertFalse(check_result_ignore(result, ranges)) ranges.append(([], SourceRange.from_values("e", 2, 3, 3, 3))) self.assertFalse(check_result_ignore(result, ranges)) ranges.append(([], SourceRange.from_values("e", 1, 1, 2, 2))) self.assertTrue(check_result_ignore(result, ranges)) result1 = Result.from_values("origin", "message", file="e") self.assertFalse(check_result_ignore(result1, ranges)) ranges = [(['something', 'else', 'not origin'], SourceRange.from_values("e", 1, 1, 2, 2))] self.assertFalse(check_result_ignore(result, ranges)) ranges = [(['something', 'else', 'origin'], SourceRange.from_values("e", 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges))
def test_ignore_results(self): ranges = [([], SourceRange.from_values('f', 1, 1, 2, 2))] result = Result.from_values('origin (Something Specific)', 'message', file='e', line=1, column=1, end_line=2, end_column=2) self.assertFalse(check_result_ignore(result, ranges)) ranges.append(([], SourceRange.from_values('e', 2, 3, 3, 3))) self.assertFalse(check_result_ignore(result, ranges)) ranges.append(([], SourceRange.from_values('e', 1, 1, 2, 2))) self.assertTrue(check_result_ignore(result, ranges)) result1 = Result.from_values('origin', 'message', file='e') self.assertTrue(check_result_ignore(result1, ranges)) ranges = [(['something', 'else', 'not origin'], SourceRange.from_values('e', 1, 1, 2, 2))] self.assertFalse(check_result_ignore(result, ranges)) ranges = [(['something', 'else', 'origin'], SourceRange.from_values('e', 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges))
def test_ignore_glob(self): result = Result.from_values("LineLengthBear", "message", file="d", line=1, column=1, end_line=2, end_column=2) ranges = [(["(line*|space*)", "py*"], SourceRange.from_values("d", 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges)) result = Result.from_values("SpaceConsistencyBear", "message", file="d", line=1, column=1, end_line=2, end_column=2) ranges = [(["(line*|space*)", "py*"], SourceRange.from_values("d", 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges)) result = Result.from_values("XMLBear", "message", file="d", line=1, column=1, end_line=2, end_column=2) ranges = [(["(line*|space*)", "py*"], SourceRange.from_values("d", 1, 1, 2, 2))] self.assertFalse(check_result_ignore(result, ranges))
def test_overlaps(self): overlapping_range = SourceRange.from_values("file1", 1, 1, 2, 2) nonoverlapping_range = SourceRange.from_values("file2", 1, 1, 2, 2) uut = Result.from_values("origin", "message", file="file1", line=1, column=1, end_line=2, end_column=2) self.assertTrue(uut.overlaps(overlapping_range)) self.assertTrue(uut.overlaps([overlapping_range])) self.assertFalse(uut.overlaps(nonoverlapping_range))
def setUp(self): self.section = Section('') self.uut = QuotesBear(self.section, Queue()) self.double_quote_file = dedent(""" ''' Multiline string ''' "a string with double quotes!" 'A single quoted string with " in it' """).splitlines(True) self.single_quote_file = dedent(""" ''' Multiline string ''' 'a string with single quotes!' "A double quoted string with ' in it" """).splitlines(True) self.filename = 'f' self.dep_results = { 'AnnotationBear': [HiddenResult( 'AnnotationBear', {'comments': (), 'strings': ( SourceRange.from_values(self.filename, 2, 1, 4, 3), SourceRange.from_values(self.filename, 5, 1, 5, 30), SourceRange.from_values(self.filename, 6, 1, 6, 37)) } )] }
def test_print_results_multiple_ranges(self): affected_code = ( SourceRange.from_values("some_file", 5, end_line=7), SourceRange.from_values("another_file", 1, 3, 1, 5), SourceRange.from_values("another_file", 3, 3, 3, 5)) with retrieve_stdout() as stdout: print_results( self.log_printer, Section(""), [Result("ClangCloneDetectionBear", "Clone Found", affected_code)], {abspath("some_file"): ["line " + str(i + 1) + "\n" for i in range(10)], abspath("another_file"): ["line " + str(i + 1) + "\n" for i in range(10)]}, {}, color=False) self.assertEqual(""" another_file | 1| line 1 another_file | 3| line 3 some_file | 5| line 5 | 6| line 6 | 7| line 7 | | [NORMAL] ClangCloneDetectionBear: | | Clone Found """, stdout.getvalue())
def yield_ignore_ranges(file_dict): """ Yields tuples of affected bears and a SourceRange that shall be ignored for those. :param file_dict: The file dictionary. """ for filename, file in file_dict.items(): start = None bears = [] for line_number, line in enumerate(file, start=1): line = line.lower() if "start ignoring " in line: start = line_number bears = get_ignore_scope(line, "start ignoring ") elif "stop ignoring" in line: if start: yield (bears, SourceRange.from_values(filename, start, end_line=line_number)) elif "ignore " in line: yield (get_ignore_scope(line, "ignore "), SourceRange.from_values(filename, line_number, end_line=line_number+1))
def test_overlaps(self): overlapping_range = SourceRange.from_values('file1', 1, 1, 2, 2) nonoverlapping_range = SourceRange.from_values('file2', 1, 1, 2, 2) uut = Result.from_values('origin', 'message', file='file1', line=1, column=1, end_line=2, end_column=2) self.assertTrue(uut.overlaps(overlapping_range)) self.assertTrue(uut.overlaps([overlapping_range])) self.assertFalse(uut.overlaps(nonoverlapping_range)) overlapping_range = SourceRange.from_values('file1', 1, None, 1, None) nonoverlapping_range = SourceRange.from_values( 'file2', 1, None, 1, None) uut = Result.from_values('origin', 'message', file='file1', line=1, column=1, end_line=1, end_column=20) self.assertTrue(uut.overlaps(overlapping_range)) self.assertTrue(uut.overlaps([overlapping_range])) self.assertFalse(uut.overlaps(nonoverlapping_range))
def test_ignore_glob(self): result = Result.from_values('LineLengthBear', 'message', file='d', line=1, column=1, end_line=2, end_column=2) ranges = [(['(line*|space*)', 'py*'], SourceRange.from_values('d', 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges)) result = Result.from_values('SpaceConsistencyBear', 'message', file='d', line=1, column=1, end_line=2, end_column=2) ranges = [(['(line*|space*)', 'py*'], SourceRange.from_values('d', 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges)) result = Result.from_values('XMLBear', 'message', file='d', line=1, column=1, end_line=2, end_column=2) ranges = [(['(line*|space*)', 'py*'], SourceRange.from_values('d', 1, 1, 2, 2))] self.assertFalse(check_result_ignore(result, ranges))
def test_ignore_results(self): ranges = [([], SourceRange.from_values('f', 1, 1, 2, 2))] result = Result.from_values('origin (Something Specific)', 'message', file='e', line=1, column=1, end_line=2, end_column=2) self.assertFalse(check_result_ignore(result, ranges)) ranges.append(([], SourceRange.from_values('e', 2, 3, 3, 3))) self.assertFalse(check_result_ignore(result, ranges)) ranges.append(([], SourceRange.from_values('e', 1, 1, 2, 2))) self.assertTrue(check_result_ignore(result, ranges)) result1 = Result.from_values('origin', 'message', file='e') self.assertTrue(check_result_ignore(result1, ranges)) ranges = [(['something', 'else', 'not origin'], SourceRange.from_values('e', 1, 1, 2, 2))] self.assertFalse(check_result_ignore(result, ranges)) ranges = [(['something', 'else', 'origin'], SourceRange.from_values('e', 1, 1, 2, 2))] self.assertTrue(check_result_ignore(result, ranges))
def test_keyword_diff(self): text = ['# todo 123\n'] comments = [SourceRange.from_values('F', 1, 1, 1, 10)] dep_results = { 'AnnotationBear': [ self.annotation_bear_result_type({'comments': comments}) ] } with execute_bear(self.uut, 'F', text, dependency_results=dep_results) as result: self.assertEqual(result[0].diffs['F'].unified_diff, '--- \n' '+++ \n' '@@ -1 +0,0 @@\n' '-# todo 123\n') text = ['test = 55 # todo 123\n'] comments = [SourceRange.from_values('F', 1, 11, 1, 23)] dep_results = { 'AnnotationBear': [ self.annotation_bear_result_type({'comments': comments}) ] } with execute_bear(self.uut, 'F', text, dependency_results=dep_results) as result: self.assertEqual(result[0].diffs['F'].unified_diff, '--- \n' '+++ \n' '@@ -1 +1 @@\n' '-test = 55 # todo 123\n' '+test = 55\n')
def test_keyword_between_code(self): self.section.append(Setting('language', 'c')) self.section.append(Setting('keywords', 'todo')) text = ['int a=0; /* TODO: Test */ int b=1;\n'] comments = [SourceRange.from_values('F', 1, 10, 1, 25)] dep_results = { 'AnnotationBear': [self.annotation_bear_result_type({'comments': comments})] } with execute_bear(self.uut, 'F', text, dependency_results=dep_results) as result: self.assertEqual( result[0].diffs['F'].unified_diff, '--- \n' '+++ \n' '@@ -1 +1 @@\n' '-int a=0; /* TODO: Test */ int b=1;\n' '+int a=0; int b=1;\n') text = ['int a = 0; /* TODO test\n', 'another test\n', '*/\n'] comments = [SourceRange.from_values('F', 1, 12, 3, 2)] dep_results = { 'AnnotationBear': [self.annotation_bear_result_type({'comments': comments})] } with execute_bear(self.uut, 'F', text, dependency_results=dep_results) as result: self.assertEqual( result[0].diffs['F'].unified_diff, '--- \n' '+++ \n' '@@ -1,3 +1,3 @@\n' '-int a = 0; /* TODO test\n' '+int a = 0; /*\n' ' another test\n' ' */\n') text = ['/* TODO\n', 'test\n', '*/\n'] comments = [SourceRange.from_values('F', 1, 1, 3, 2)] dep_results = { 'AnnotationBear': [self.annotation_bear_result_type({'comments': comments})] } with execute_bear(self.uut, 'F', text, dependency_results=dep_results) as result: self.assertEqual( result[0].diffs['F'].unified_diff, '--- \n' '+++ \n' '@@ -1,3 +1,3 @@\n' '-/* TODO\n' '+/*\n' ' test\n' ' */\n')
def test_no_overlap(self): uut1 = SourceRange.from_values('file', 2, None, 3) uut2 = SourceRange.from_values('file', 4, None, 5) self.assertFalse(uut1.overlaps(uut2)) self.assertFalse(uut2.overlaps(uut1)) uut1 = SourceRange.from_values('file', 2, None, 3, 6) uut2 = SourceRange.from_values('file', 3, 7, 5) self.assertFalse(uut1.overlaps(uut2)) self.assertFalse(uut2.overlaps(uut1))
def test_construction(self): uut1 = SourceRange(self.result_fileA_noline) self.assertEqual(uut1.end, self.result_fileA_noline) uut2 = SourceRange.from_values("A") self.assertEqual(uut1, uut2) uut = SourceRange.from_values("B", start_line=2, end_line=4) self.assertEqual(uut.start, self.result_fileB_line2) self.assertEqual(uut.end, self.result_fileB_line4)
def test_no_overlap(self): uut1 = SourceRange.from_values('file', 2, None, 3) uut2 = SourceRange.from_values('file', 4, None, 5) self.assertFalse(uut1.overlaps(uut2)) self.assertFalse(uut2.overlaps(uut1)) uut1 = SourceRange.from_values('file', 2, None, 3, 6) uut2 = SourceRange.from_values('file', 3, 7, 5) self.assertFalse(uut1.overlaps(uut2)) self.assertFalse(uut2.overlaps(uut1))
def test_construction(self): uut1 = SourceRange(self.result_fileA_noline) self.assertEqual(uut1.end, self.result_fileA_noline) uut2 = SourceRange.from_values('A') self.assertEqual(uut1, uut2) uut = SourceRange.from_values('B', start_line=2, end_line=4) self.assertEqual(uut.start, self.result_fileB_line2) self.assertEqual(uut.end, self.result_fileB_line4)
def yield_ignore_ranges(file_dict): """ Yields tuples of affected bears and a SourceRange that shall be ignored for those. :param file_dict: The file dictionary. """ for filename, file in file_dict.items(): start = None bears = [] stop_ignoring = False # Do not process raw files if file is None: continue for line_number, line in enumerate(file, start=1): # Before lowering all lines ever read, first look for the biggest # common substring, case sensitive: I*gnor*e, start i*gnor*ing, # N*oqa*. if 'gnor' in line or 'oqa' in line: line = line.lower() if 'start ignoring ' in line: start = line_number bears = get_ignore_scope(line, 'start ignoring ') elif 'stop ignoring' in line: stop_ignoring = True if start: yield (bears, SourceRange.from_values( filename, start, 1, line_number, len(file[line_number-1]))) else: for ignore_stmt in ['ignore ', 'noqa ', 'noqa']: if ignore_stmt in line: end_line = min(line_number + 1, len(file)) yield (get_ignore_scope(line, ignore_stmt), SourceRange.from_values( filename, line_number, 1, end_line, len(file[end_line-1]))) break if stop_ignoring is False and start is not None: yield (bears, SourceRange.from_values(filename, start, 1, len(file), len(file[-1])))
def yield_ignore_ranges(file_dict): """ Yields tuples of affected bears and a SourceRange that shall be ignored for those. :param file_dict: The file dictionary. """ for filename, file in file_dict.items(): start = None bears = [] stop_ignoring = False # Do not process raw files if file is None: continue for line_number, line in enumerate(file, start=1): # Before lowering all lines ever read, first look for the biggest # common substring, case sensitive: I*gnor*e, start i*gnor*ing, # N*oqa*. if 'gnor' in line or 'oqa' in line: line = line.lower() if 'start ignoring ' in line: start = line_number bears = get_ignore_scope(line, 'start ignoring ') elif 'stop ignoring' in line: stop_ignoring = True if start: yield (bears, SourceRange.from_values( filename, start, 1, line_number, len(file[line_number-1]))) else: for ignore_stmt in ['ignore ', 'noqa ', 'noqa']: if ignore_stmt in line: end_line = min(line_number + 1, len(file)) yield (get_ignore_scope(line, ignore_stmt), SourceRange.from_values( filename, line_number, 1, end_line, len(file[end_line-1]))) break if stop_ignoring is False and start is not None: yield (bears, SourceRange.from_values(filename, start, 1, len(file), len(file[-1])))
def test_overlaps(self): overlapping_range = SourceRange.from_values("file1", 1, 1, 2, 2) nonoverlapping_range = SourceRange.from_values("file2", 1, 1, 2, 2) uut = Result.from_values("origin", "message", file="file1", line=1, column=1, end_line=2, end_column=2) self.assertTrue(uut.overlaps(overlapping_range)) self.assertTrue(uut.overlaps([overlapping_range])) self.assertFalse(uut.overlaps(nonoverlapping_range))
def test_overlaps(self): overlapping_range = SourceRange.from_values('file1', 1, 1, 2, 2) nonoverlapping_range = SourceRange.from_values('file2', 1, 1, 2, 2) uut = Result.from_values('origin', 'message', file='file1', line=1, column=1, end_line=2, end_column=2) self.assertTrue(uut.overlaps(overlapping_range)) self.assertTrue(uut.overlaps([overlapping_range])) self.assertFalse(uut.overlaps(nonoverlapping_range))
def test_multiple_ranges(self): expected_string = ("id:-?[0-9]+:origin:1:.*file:.*another_file:line:5:" "column:3:end_line:5:end_column:5:" "severity:1:severity_str:NORMAL:message:2\n" "id:-?[0-9]+:origin:1:.*file:.*some_file:line:5:" "column:None:end_line:7:end_column:None:" "severity:1:severity_str:NORMAL:message:2\n") affected_code = (SourceRange.from_values("some_file", 5, end_line=7), SourceRange.from_values("another_file", 5, 3, 5, 5)) with retrieve_stdout() as stdout: print_results_formatted(self.logger, self.section, [Result("1", "2", affected_code)], None, None) self.assertRegex(stdout.getvalue(), expected_string)
def test_location_repr(self): result_a = Result(origin="o", message="m") self.assertEqual(result_a.location_repr(), "the whole project") result_b = Result.from_values("o", "m", file="e") self.assertEqual(result_b.location_repr(), "'e'") affected_code = (SourceRange.from_values("f"), SourceRange.from_values("g")) result_c = Result("o", "m", affected_code=affected_code) self.assertEqual(result_c.location_repr(), "'f', 'g'") affected_code = (SourceRange.from_values("f"), SourceRange.from_values("f")) result_d = Result("o", "m", affected_code=affected_code) self.assertEqual(result_d.location_repr(), "'f'")
def test_nostdin_nostderr_noconfig_correction(self): create_arguments_mock = Mock() class Handler: @staticmethod def create_arguments(filename, file, config_file): create_arguments_mock(filename, file, config_file) return self.test_program_path, '--correct', filename uut = (linter(sys.executable, output_format='corrected', diff_severity=RESULT_SEVERITY.INFO, result_message='Custom message')(Handler)(self.section, None)) results = list(uut.run(self.testfile_path, self.testfile_content)) expected_correction = [ s + '\n' for s in ['+', '-', '*', '++', '-', '-', '+'] ] diffs = list( Diff.from_string_arrays(self.testfile_content, expected_correction).split_diff()) expected = [ Result(uut, 'Custom message', affected_code=(SourceRange.from_values( self.testfile_path, 4), SourceRange.from_values(self.testfile_path, 6)), severity=RESULT_SEVERITY.INFO, diffs={self.testfile_path: diffs[0]}), Result.from_values(uut, 'Custom message', self.testfile_path, 10, None, 10, None, RESULT_SEVERITY.INFO, diffs={self.testfile_path: diffs[1]}) ] self.assertEqual(results, expected) create_arguments_mock.assert_called_once_with(self.testfile_path, self.testfile_content, None)
def test_location_repr(self): result_a = Result(origin="o", message="m") self.assertEqual(result_a.location_repr(), "the whole project") result_b = Result.from_values("o", "m", file="e") self.assertEqual(result_b.location_repr(), "'e'") affected_code = (SourceRange.from_values('f'), SourceRange.from_values('g')) result_c = Result("o", "m", affected_code=affected_code) self.assertEqual(result_c.location_repr(), "'f', 'g'") affected_code = (SourceRange.from_values('f'), SourceRange.from_values('f')) result_d = Result("o", "m", affected_code=affected_code) self.assertEqual(result_d.location_repr(), "'f'")
def get_language_tool_results(filename, file_contents, locale): joined_text = "".join(file_contents) locale = guess_language(joined_text) if locale == 'auto' else locale locale = 'en-US' if not locale else locale tool = LanguageTool(locale) matches = tool.check(joined_text) for match in matches: if not match.replacements: diffs = None else: replaced = correct(joined_text, [match]).splitlines(True) diffs = { filename: Diff.from_string_arrays(file_contents, replaced) } rule_id = match.ruleId if match.subId is not None: rule_id += '[{}]'.format(match.subId) message = match.msg + ' (' + rule_id + ')' yield message, diffs, SourceRange.from_values(filename, match.fromy + 1, match.fromx + 1, match.toy + 1, match.tox + 1)
def process_output(self, output, file, filename): outputs = json.loads(output) for message_type, values in outputs.items(): if message_type != 'warnings': continue for value in values: sourceranges = [SourceRange.from_values( file=value['file'], start_line=value['line'], end_line=value['line'])] if value['code'] is None: message = "'{}': {}".format( value['check_name'], value['message']) else: message = "'{}' (in '{}'): {}.".format( value['check_name'], value['code'], value['message']) yield Result( origin='{} ({})'.format(self.__class__.__name__, value['warning_type']), message=message, affected_code=sourceranges, severity=self.severity_map[value['confidence']], additional_info='More information is available at {}' '.'.format(value['link']))
def test_location_repr(self): result_a = Result(origin='o', message='m') self.assertEqual(result_a.location_repr(), 'the whole project') result_b = Result.from_values('o', 'm', file='e') self.assertEqual(result_b.location_repr(), "'e'") affected_code = (SourceRange.from_values('f'), SourceRange.from_values('g')) result_c = Result('o', 'm', affected_code=affected_code) self.assertEqual(result_c.location_repr(), "'f', 'g'") affected_code = (SourceRange.from_values('f'), SourceRange.from_values('f')) result_d = Result('o', 'm', affected_code=affected_code) self.assertEqual(result_d.location_repr(), "'f'")
def process_output(self, output, file, filename): if not output: # backwards compatible no results return outputs = json.loads(output) for message_type, values in outputs.items(): if message_type != 'warnings': continue for value in values: sourceranges = [SourceRange.from_values( file=value['file'], start_line=value['line'], end_line=value['line'])] if value['code'] is None: message = "'{}': {}".format( value['check_name'], value['message']) else: message = "'{}' (in '{}'): {}.".format( value['check_name'], value['code'], value['message']) yield Result( origin='{} ({})'.format(self.__class__.__name__, value['warning_type']), message=message, affected_code=sourceranges, severity=self.severity_map[value['confidence']], additional_info='More information is available at {}' '.'.format(value['link']))
def parse_output(self, out, filename): """ Parses the output JSON into Result objects. :param out: Raw output from the given executable (should be JSON). :param filename: The filename of the analyzed file. Needed to create the Result objects. :return: An iterator yielding ``Result`` objects. """ output = json.loads(out) for result in output["results"]: affected_code = tuple( SourceRange.from_values( code_range["file"], code_range["start"]["line"], code_range["start"].get("column"), code_range.get("end", {}).get("line"), code_range.get("end", {}).get("column"), ) for code_range in result["affected_code"] ) yield Result( origin=result["origin"], message=result["message"], affected_code=affected_code, severity=result.get("severity", 1), debug_msg=result.get("debug_msg", ""), additional_info=result.get("additional_info", ""), )
def parse_output(self, out, filename): """ Parses the output JSON into Result objects. :param out: Raw output from the given executable (should be JSON). :param filename: The filename of the analyzed file. Needed to create the Result objects. :return: An iterator yielding ``Result`` objects. """ output = json.loads(out) for result in output['results']: affected_code = tuple( SourceRange.from_values( code_range['file'], code_range['start']['line'], code_range['start'].get('column'), code_range.get('end', {}).get('line'), code_range.get('end', {}).get('column')) for code_range in result['affected_code']) yield Result( origin=result['origin'], message=result['message'], affected_code=affected_code, severity=result.get('severity', 1), debug_msg=result.get('debug_msg', ''), additional_info=result.get('additional_info', ''))
def test_result_range_line_wise_overlap(self): test_file = ['11', '22', '33', '44', '55', '66'] test_file_dict = {abspath('test_file'): test_file} source_range1 = SourceRange.from_values('test_file', 2, 2, 5, 1) source_range2 = SourceRange.from_values('test_file', 3, 1, 4, 1) test_result = Result('origin', 'message', (source_range1, source_range2)) result_diff = remove_result_ranges_diffs( [test_result], test_file_dict)[test_result][abspath('test_file')] expected_diff = Diff.from_string_arrays(test_file, ['11', '2', '5', '66']) self.assertEqual(result_diff, expected_diff)
def run(self, filename, file, radon_ranks_info: typed_list(str)=(), radon_ranks_normal: typed_list(str)=('C', 'D'), radon_ranks_major: typed_list(str)=('E', 'F')): """ Uses radon to compute complexity of a given file. :param radon_ranks_info: The ranks (given by radon) to treat as severity INFO. :param radon_ranks_normal: The ranks (given by radon) to treat as severity NORMAL. :param radon_ranks_major: The ranks (given by radon) to treat as severity MAJOR. """ severity_map = { RESULT_SEVERITY.INFO: radon_ranks_info, RESULT_SEVERITY.NORMAL: radon_ranks_normal, RESULT_SEVERITY.MAJOR: radon_ranks_major } for visitor in radon.complexity.cc_visit("".join(file)): rank = radon.complexity.cc_rank(visitor.complexity) severity = None for result_severity, rank_list in severity_map.items(): if rank in rank_list: severity = result_severity if severity is None: continue visitor_range = SourceRange.from_values( filename, visitor.lineno, visitor.col_offset, visitor.endline) message = "{} has a cyclomatic complexity of {}".format( visitor.name, rank) yield Result(self, message, severity=severity, affected_code=(visitor_range,))
def test_multiple_ranges(self): expected_string = ( "id:-?[0-9]+:origin:1:.*file:.*another_file:from_line:5:" "from_column:3:to_line:5:to_column:5:" "severity:1:msg:2\n" "id:-?[0-9]+:origin:1:.*file:.*some_file:from_line:5:" "from_column:None:to_line:7:to_column:None:" "severity:1:msg:2\n" ) affected_code = ( SourceRange.from_values("some_file", 5, end_line=7), SourceRange.from_values("another_file", 5, 3, 5, 5), ) with retrieve_stdout() as stdout: print_results_formatted(self.logger, self.section, [Result("1", "2", affected_code)], None, None) self.assertRegex(stdout.getvalue(), expected_string)
def generate_diff(comments, file, filename, line, line_number, pos): todo_source_range = SourceRange.from_values(filename, line_number, pos + 1) affected_comment_sourcerange = [ c for c in comments if todo_source_range in c ] affected_len = len(affected_comment_sourcerange) if affected_len == 0: return {} assert affected_len == 1, 'More than 1 affected comment source ranges' comment_sourcerange = affected_comment_sourcerange[0] comment_start = comment_sourcerange.start.column comment_end = comment_sourcerange.end.column in_multi_line_comment = (comment_sourcerange.start.line != comment_sourcerange.end.line) line_before_todo_comment = line[:comment_start - 1].rstrip() line_behind_todo_comment = line[comment_end:].rstrip() line_replacement = line_before_todo_comment + line_behind_todo_comment diff = Diff(file) if line_replacement and not in_multi_line_comment: diff.change_line(line_number, line, line_replacement + '\n') elif line_replacement and in_multi_line_comment: text_replacement = line[pos:] diff.change_line(line_number, line, line.replace(text_replacement, '').rstrip() + '\n') else: diff.delete_line(line_number) return {filename: diff}
def run(self, filename, file, network_timeout: typed_dict(str, int, DEFAULT_TIMEOUT)=dict(), link_ignore_regex: str='([.\/]example\.com|\{|\$)', link_ignore_list: typed_list(str)=''): """ Find links in any text file. Warning: This bear will make HEAD requests to all URLs mentioned in your codebase, which can potentially be destructive. As an example, this bear would naively just visit the URL from a line that goes like `do_not_ever_open = 'https://api.acme.inc/delete-all-data'` wiping out all your data. :param network_timeout: A dict mapping URLs and timeout to be used for that URL. All the URLs that have the same host as that of URLs provided will be passed that timeout. It can also contain a wildcard timeout entry with key '*'. The timeout of all the websites not in the dict will be the value of the key '*'. :param link_ignore_regex: A regex for urls to ignore. :param link_ignore_list: Comma separated url globs to ignore """ network_timeout = {urlparse(url).netloc if not url == '*' else '*': timeout for url, timeout in network_timeout.items()} for line_number, link, code, context in self.analyze_links_in_file( file, network_timeout, link_ignore_regex, link_ignore_list): affected_code = SourceRange.from_values(filename, line_number) yield URLResult(self, (affected_code,), link, code, context)
def test_result_range_inline_overlap(self): test_file = ["123456789\n"] test_file_dict = {abspath("test_file"): test_file} source_range1 = SourceRange.from_values("test_file", 1, 1, 1, 4) source_range2 = SourceRange.from_values("test_file", 1, 2, 1, 3) source_range3 = SourceRange.from_values("test_file", 1, 3, 1, 6) test_result = Result("origin", "message", (source_range1, source_range2, source_range3)) result_diff = remove_result_ranges_diffs( [test_result], test_file_dict)[test_result][abspath("test_file")] expected_diff = Diff.from_string_arrays(test_file, ["789\n"]) self.assertEqual(result_diff, expected_diff)
def test_result_range_line_wise_overlap(self): test_file = ["11", "22", "33", "44", "55", "66"] test_file_dict = {abspath("test_file"): test_file} source_range1 = SourceRange.from_values("test_file", 2, 2, 5, 1) source_range2 = SourceRange.from_values("test_file", 3, 1, 4, 1) test_result = Result("origin", "message", (source_range1, source_range2)) result_diff = remove_result_ranges_diffs( [test_result], test_file_dict)[test_result][abspath("test_file")] expected_diff = Diff.from_string_arrays(test_file, ["11", "2", "5", "66"]) self.assertEqual(result_diff, expected_diff)
def run(self, filename, file, radon_ranks_info: typed_list(str)=(), radon_ranks_normal: typed_list(str)=('C', 'D'), radon_ranks_major: typed_list(str)=('E', 'F')): """ Uses radon to compute complexity of a given file. :param radon_ranks_info: The ranks (given by radon) to treat as severity INFO. :param radon_ranks_normal: The ranks (given by radon) to treat as severity NORMAL. :param radon_ranks_major: The ranks (given by radon) to treat as severity MAJOR. """ severity_map = { RESULT_SEVERITY.INFO: radon_ranks_info, RESULT_SEVERITY.NORMAL: radon_ranks_normal, RESULT_SEVERITY.MAJOR: radon_ranks_major } for visitor in radon.complexity.cc_visit("".join(file)): rank = radon.complexity.cc_rank(visitor.complexity) severity = None for result_severity, rank_list in severity_map.items(): if rank in rank_list: severity = result_severity if severity is None: continue visitor_range = SourceRange.from_values( filename, visitor.lineno, visitor.col_offset, visitor.endline) message = "{} has a cyclomatic complexity of {}".format( visitor.name, rank) yield Result(self, message, severity=severity, affected_code=(visitor_range,))
def get_language_tool_results(filename, file_contents, locale): joined_text = "".join(file_contents) locale = guess_language(joined_text) if locale == 'auto' else locale locale = 'en-US' if not locale else locale tool = LanguageTool(locale) matches = tool.check(joined_text) for match in matches: if not match.replacements: diffs = None else: replaced = correct(joined_text, [match]).splitlines(True) diffs = {filename: Diff.from_string_arrays(file_contents, replaced)} rule_id = match.ruleId if match.subId is not None: rule_id += '[{}]'.format(match.subId) message = match.msg + ' (' + rule_id + ')' yield message, diffs, SourceRange.from_values(filename, match.fromy+1, match.fromx+1, match.toy+1, match.tox+1)
def test_result_range_inline_overlap(self): test_file = ['123456789\n'] test_file_dict = {abspath('test_file'): test_file} source_range1 = SourceRange.from_values('test_file', 1, 1, 1, 4) source_range2 = SourceRange.from_values('test_file', 1, 2, 1, 3) source_range3 = SourceRange.from_values('test_file', 1, 3, 1, 6) test_result = Result('origin', 'message', (source_range1, source_range2, source_range3)) result_diff = remove_result_ranges_diffs( [test_result], test_file_dict)[test_result][abspath('test_file')] expected_diff = Diff.from_string_arrays(test_file, ['789\n']) self.assertEqual(result_diff, expected_diff)
def parse_output(self, out, filename): """ Parses the output JSON into Result objects. :param out: Raw output from the given executable (should be JSON). :param filename: The filename of the analyzed file. Needed to create the Result objects. :return: An iterator yielding ``Result`` objects. """ output = json.loads(out) for result in output['results']: affected_code = tuple( SourceRange.from_values( code_range['file'], code_range['start']['line'], code_range['start'].get('column'), code_range.get('end', {}).get('line'), code_range.get('end', {}).get('column')) for code_range in result['affected_code']) yield Result(origin=result['origin'], message=result['message'], affected_code=affected_code, severity=result.get('severity', 1), debug_msg=result.get('debug_msg', ""), additional_info=result.get('additional_info', ""))
def test_process_output(self): section = Section("some_name") self.uut = Lint(section, None) out = list(self.uut.process_output( "1.0|0: Info message\n" "2.2|1: Normal message\n" "3.4|2: Major message\n", "a/file.py")) self.assertEqual(len(out), 3) self.assertEqual(out[0].origin, "Lint") self.assertEqual(out[0].affected_code[0], SourceRange.from_values("a/file.py", 1, 0)) self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO) self.assertEqual(out[0].message, "Info message") self.assertEqual(out[1].affected_code[0], SourceRange.from_values("a/file.py", 2, 2)) self.assertEqual(out[1].severity, RESULT_SEVERITY.NORMAL) self.assertEqual(out[1].message, "Normal message") self.assertEqual(out[2].affected_code[0], SourceRange.from_values("a/file.py", 3, 4)) self.assertEqual(out[2].severity, RESULT_SEVERITY.MAJOR) self.assertEqual(out[2].message, "Major message") self.uut = Lint(section, None) self.uut.output_regex = (r'(?P<line>\d+)\.(?P<column>\d+)\|' r'(?P<end_line>\d+)\.(?P<end_column>\d+)\|' r'(?P<severity>\d+): (?P<message>.*)') self.uut.severity_map = {"I": RESULT_SEVERITY.INFO} out = list(self.uut.process_output( "1.0|2.3|0: Info message\n", 'a/file.py')) self.assertEqual(len(out), 1) self.assertEqual(out[0].affected_code[0].start.line, 1) self.assertEqual(out[0].affected_code[0].start.column, 0) self.assertEqual(out[0].affected_code[0].end.line, 2) self.assertEqual(out[0].affected_code[0].end.column, 3) self.assertEqual(out[0].severity, RESULT_SEVERITY.INFO) self.uut = Lint(section, None) out = list(self.uut.process_output( "Random line that shouldn't be captured\n" "*************\n", 'a/file.py')) self.assertEqual(len(out), 0)
def run(self, filename, file, natural_language: str = 'auto', languagetool_disable_rules: typed_list(str) = (), ): """ Checks the code with LanguageTool. :param natural_language: A locale representing the language you want to have checked. If set to 'auto' the language is guessed. If the language cannot be guessed or an unsupported language is guessed, 'en-US' is used. :param languagetool_disable_rules: List of rules to disable checks for. """ # Defer import so the check_prerequisites can be run without # language_check being there. from language_check import LanguageTool, correct joined_text = ''.join(file) natural_language = (guess_language(joined_text) if natural_language == 'auto' else natural_language) try: tool = LanguageTool(natural_language, motherTongue='en_US') except ValueError: # Using 'en-US' if guessed language is not supported logging.warn( "Changing the `natural_language` setting to 'en-US' as " '`language_check` failed to guess a valid language.' ) natural_language = 'en-US' tool = LanguageTool(natural_language, motherTongue='en_US') tool.disabled.update(languagetool_disable_rules) matches = tool.check(joined_text) for match in matches: if not match.replacements: diffs = None else: replaced = correct(joined_text, [match]).splitlines(True) diffs = {filename: Diff.from_string_arrays(file, replaced)} rule_id = match.ruleId if match.subId is not None: rule_id += '[{}]'.format(match.subId) message = match.msg + ' (' + rule_id + ')' source_range = SourceRange.from_values(filename, match.fromy+1, match.fromx+1, match.toy+1, match.tox+1) yield Result(self, message, diffs=diffs, affected_code=(source_range,))
def test_multiple_ranges(self): expected_string = ( 'id:-?[0-9]+:origin:1:.*file:.*another_file:line:5:' 'column:3:end_line:5:end_column:5:' 'severity:1:severity_str:NORMAL:message:2\n' 'id:-?[0-9]+:origin:1:.*file:.*some_file:line:5:' 'column:None:end_line:7:end_column:None:' 'severity:1:severity_str:NORMAL:message:2\n') affected_code = (SourceRange.from_values('some_file', 5, end_line=7), SourceRange.from_values('another_file', 5, 3, 5, 5)) with retrieve_stdout() as stdout: print_results_formatted(self.logger, self.section, [Result('1', '2', affected_code)], None, None) self.assertRegex(stdout.getvalue(), expected_string)
def test_print_results_multiple_ranges(self): affected_code = ( SourceRange.from_values('some_file', 5, end_line=7), SourceRange.from_values('another_file', 1, 3, 1, 5), SourceRange.from_values('another_file', 3, 3, 3, 5)) with retrieve_stdout() as stdout: print_results( self.log_printer, Section(''), [Result('Bear_for_detecting_clone', 'Clone Found', affected_code)], {abspath('some_file'): ['line ' + str(i + 1) + '\n' for i in range(10)], abspath('another_file'): ['line ' + str(i + 1) for i in range(10)]}, {}, self.console_printer) self.assertEqual(""" another_file [ 1] li{0}{1} another_file [ 3] li{0}{2} some_file [ 5] li{0}{3} [ 6] li{0}{4} [ 7] li{0}{5} **** Bear_for_detecting_clone [Section: | Severity: NORMAL] **** ! ! {6}\n""".format(highlight_text(self.no_color, 'ne', BackgroundSourceRangeStyle, self.lexer), highlight_text(self.no_color, ' 1', NoColorStyle, self.lexer), highlight_text(self.no_color, ' 3', NoColorStyle, self.lexer), highlight_text(self.no_color, ' 5', NoColorStyle, self.lexer), highlight_text(self.no_color, ' 6', NoColorStyle, self.lexer), highlight_text(self.no_color, ' 7', NoColorStyle, self.lexer), highlight_text(self.no_color, 'Clone Found', style=BackgroundMessageStyle), ' '), stdout.getvalue())
def test_result_range_line_wise_overlap(self): test_file = ['11', '22', '33', '44', '55', '66'] test_file_dict = {abspath('test_file'): test_file} source_range1 = SourceRange.from_values('test_file', 2, 2, 5, 1) source_range2 = SourceRange.from_values('test_file', 3, 1, 4, 1) test_result = Result('origin', 'message', (source_range1, source_range2)) result_diff = remove_result_ranges_diffs( [test_result], test_file_dict)[test_result][abspath('test_file')] expected_diff = Diff.from_string_arrays(test_file, ['11', '2', '5', '66']) self.assertEqual(result_diff, expected_diff)