def test_clang50_trunk_plist(self): """ Check plist generated by clang 5.0 trunk checker name and report hash should be in the plist file. """ clang50_trunk_plist = os.path.join(self.__plist_test_files, 'clang-5.0-trunk.plist') files, reports = plist_parser.parse_plist(clang50_trunk_plist, None, False) self.assertEquals(files, self.__found_file_names) self.assertEquals(len(reports), 3) valid_checker_names = [] valid_checker_names.extend(self.__found_checker_names) for report in reports: checker_name = report.main['check_name'] # Checker name should be in the plist file. self.assertNotEqual(checker_name, 'NOT FOUND') self.assertIn(checker_name, valid_checker_names) if checker_name == 'core.DivideZero': # Test data is still valid for this version. self.assertEqual(report.main, div_zero_skel_name_hash) if checker_name == 'core.StackAddressEscape': self.assertEqual(report.main, stack_addr_skel_name_hash_after_v40)
def get_report_dir_results(reportdir): all_reports = [] processed_path_hashes = set() for filename in os.listdir(reportdir): if filename.endswith(".plist"): file_path = os.path.join(reportdir, filename) LOG.debug("Parsing: %s", file_path) try: files, reports = plist_parser.parse_plist(file_path) for report in reports: path_hash = get_report_path_hash(report, files) if path_hash in processed_path_hashes: LOG.debug("Not showing report because it is a " "deduplication of an already processed " "report!") LOG.debug("Path hash: %s", path_hash) LOG.debug(report) continue if skip_report_dir_result(report): continue processed_path_hashes.add(path_hash) report.main['location']['file_name'] = \ files[int(report.main['location']['file'])] all_reports.append(report) except Exception as ex: LOG.error('The generated plist is not valid!') LOG.error(ex) return all_reports
def test_no_bug_file(self): """There was no bug in the checked file.""" no_bug_plist = os.path.join(self.__plist_test_files, 'clang-3.7-noerror.plist') files, reports = plist_parser.parse_plist(no_bug_plist, None, False) self.assertEquals(files, []) self.assertEquals(reports, [])
def get_report_dir_results(reportdir): all_reports = [] processed_path_hashes = set() for filename in os.listdir(reportdir): if filename.endswith(".plist"): file_path = os.path.join(reportdir, filename) LOG.debug("Parsing: %s", file_path) try: files, reports = plist_parser.parse_plist(file_path) for report in reports: path_hash = get_report_path_hash(report, files) if path_hash in processed_path_hashes: LOG.debug("Not showing report because it is a " "deduplication of an already processed " "report!") LOG.debug("Path hash: %s", path_hash) LOG.debug(report) continue if skip_report_dir_result(report): continue processed_path_hashes.add(path_hash) report.main['location']['file_name'] = \ files[int(report.main['location']['file'])] all_reports.append(report) except Exception as ex: LOG.error('The generated plist is not valid!') LOG.error(ex) return all_reports
def test_report_path_hash_generation(self): """ Test report path hash generation. """ clang50_trunk_plist = os.path.join(self.__plist_test_files, 'clang-5.0-trunk.plist') files, reports = plist_parser.parse_plist(clang50_trunk_plist, None, False) self.assertEqual(len(reports), 3) # Generate dummy file_ids which should come from the database. file_ids = {} for i, file_name in enumerate(files, 1): file_ids[file_name] = i msg = "This test is prepared to handle 3 reports." self.assertEqual(len(reports), 3, msg) report_hash_to_path_hash = { '79e31a6ba028f0b7d9779faf4a6cb9cf': 'c473c1a55df72ea4c6e055e18370ac65', '8714f42d8328bc78d5d7bff6ced918cc': '94f2a6eee8af6462a810218dff35056a', 'a6d3464f8aab9eb31a8ea7e167e84322': '11f410136724cf43c63526841007897e' } for report in reports: path_hash = get_report_path_hash(report, files) bug_hash = report.main['issue_hash_content_of_line_in_context'] self.assertEqual(path_hash, report_hash_to_path_hash[bug_hash])
def collect_file_hashes_from_plist(plist_file): """ Collects file content hashes and last modification times of files which can be found in the given plist file. :returns List of file paths which are in the processed plist file but missing from the user's disk. """ missing_files = [] try: files, _ = plist_parser.parse_plist(plist_file) for f in files: if not os.path.isfile(f): missing_files.append(f) missing_source_files.add(f) continue content_hash = util.get_file_content_hash(f) hash_to_file[content_hash] = f file_to_hash[f] = content_hash file_to_mtime[f] = util.get_last_mod_time(f) return missing_files except Exception as ex: LOG.error('Parsing the plist failed: %s', str(ex))
def test_clang37_plist(self): """ Check plist generated by clang 3.7 checker name should be in the plist file generating a report hash is still needed. """ clang37_plist = os.path.join(self.__plist_test_files, 'clang-3.7.plist') files, reports = plist_parser.parse_plist(clang37_plist, None, False) self.assertEquals(files, self.__found_file_names) self.assertEquals(len(reports), 3) self.missing_hash(reports)
def collect_file_hashes_from_plist(plist_file): """ Collects file content hashes and last modification times for the source files which can be found in the given plist file. :returns List of file paths which are in the processed plist file but missing from the user's disk and the source file modification times for the still available source files. """ source_file_mod_times = {} missing_files = [] try: files, reports = plist_parser.parse_plist(plist_file) for f in files: if not os.path.isfile(f): missing_files.append(f) missing_source_files.add(f) continue content_hash = get_file_content_hash(f) hash_to_file[content_hash] = f file_to_hash[f] = content_hash source_file_mod_times[f] = util.get_last_mod_time(f) # Get file hashes which contain source code comments. for report in reports: last_report_event = report.bug_path[-1] file_path = files[last_report_event['location']['file']] if not os.path.isfile(file_path): continue file_hash = file_to_hash[file_path] if file_hash in file_hash_with_review_status: continue report_line = last_report_event['location']['line'] sc_handler = SourceCodeCommentHandler(file_path) if sc_handler.has_source_line_comments(report_line): file_hash_with_review_status.add(file_hash) return missing_files, source_file_mod_times except Exception as ex: LOG.error('Parsing the plist failed: %s', str(ex))
def collect_file_hashes_from_plist(plist_file): """ Collects file content hashes and last modification times of files which can be found in the given plist file. :returns List of file paths which are in the processed plist file but missing from the user's disk. """ missing_files = [] try: files, reports = plist_parser.parse_plist(plist_file) for f in files: if not os.path.isfile(f): missing_files.append(f) missing_source_files.add(f) continue content_hash = util.get_file_content_hash(f) hash_to_file[content_hash] = f file_to_hash[f] = content_hash file_to_mtime[f] = util.get_last_mod_time(f) # Get file hashes which contain source code comments. for report in reports: last_report_event = report.bug_path[-1] file_path = files[last_report_event['location']['file']] if not os.path.isfile(file_path): continue file_hash = file_to_hash[file_path] if file_hash in file_hash_with_review_status: continue report_line = last_report_event['location']['line'] sc_handler = SourceCodeCommentHandler(file_path) if sc_handler.has_source_line_comments(report_line): file_hash_with_review_status.add(file_hash) return missing_files except Exception as ex: LOG.error('Parsing the plist failed: %s', str(ex))
def test_collect_path_events(self): """ Test path event collect before store. """ clang50_trunk_plist = os.path.join(self.__plist_test_files, 'clang-5.0-trunk.plist') files, reports = plist_parser.parse_plist(clang50_trunk_plist, None, False) self.assertEqual(len(reports), 3) # Generate dummy file_ids which should come from the database. file_ids = {} for i, file_name in enumerate(files, 1): file_ids[file_name] = i msg = "This test is prepared to handle 3 reports." self.assertEqual(len(reports), 3, msg) report1_path = [ ttypes.BugPathPos(startLine=19, filePath=None, endCol=7, startCol=5, endLine=19, fileId=1), ttypes.BugPathPos(startLine=20, filePath=None, endCol=7, startCol=5, endLine=20, fileId=1), ttypes.BugPathPos(startLine=21, filePath=None, endCol=13, startCol=5, endLine=21, fileId=1), ttypes.BugPathPos(startLine=7, filePath=None, endCol=7, startCol=5, endLine=7, fileId=1), ttypes.BugPathPos(startLine=8, filePath=None, endCol=6, startCol=5, endLine=8, fileId=1), ttypes.BugPathPos(startLine=8, filePath=None, endCol=25, startCol=22, endLine=8, fileId=1), ttypes.BugPathPos(startLine=8, filePath=None, endCol=20, startCol=10, endLine=8, fileId=1), ttypes.BugPathPos(startLine=7, filePath=None, endCol=14, startCol=14, endLine=7, fileId=2) ] report1_events = [ ttypes.BugPathEvent(startLine=20, filePath=None, endCol=12, startCol=5, msg="'base' initialized to 0", endLine=20, fileId=1), ttypes.BugPathEvent(startLine=21, filePath=None, endCol=18, startCol=15, msg="Passing the value 0 via " "1st parameter 'base'", endLine=21, fileId=1), ttypes.BugPathEvent(startLine=21, filePath=None, endCol=19, startCol=5, msg="Calling 'test_func'", endLine=21, fileId=1), ttypes.BugPathEvent(startLine=6, filePath=None, endCol=1, startCol=1, msg="Entered call from 'main'", endLine=6, fileId=1), ttypes.BugPathEvent(startLine=8, filePath=None, endCol=25, startCol=22, msg="Passing the value 0 via " "1st parameter 'num'", endLine=8, fileId=1), ttypes.BugPathEvent(startLine=8, filePath=None, endCol=26, startCol=10, msg="Calling 'generate_id'", endLine=8, fileId=1), ttypes.BugPathEvent(startLine=6, filePath=None, endCol=1, startCol=1, msg="Entered call from 'test_func'", endLine=6, fileId=2), ttypes.BugPathEvent(startLine=7, filePath=None, endCol=17, startCol=12, msg='Division by zero', endLine=7, fileId=2) ] path1, events1, _ = store_handler.collect_paths_events( reports[0], file_ids, files) self.assertEqual(path1, report1_path) self.assertEqual(events1, report1_events) report2_path = [] report2_events = [ ttypes.BugPathEvent(startLine=8, filePath=None, endCol=26, startCol=10, msg="Value stored to 'id' is " "never read", endLine=8, fileId=1) ] path2, events2, _ = store_handler.collect_paths_events( reports[1], file_ids, files) self.assertEqual(path2, report2_path) self.assertEqual(events2, report2_events) report3_path = [ ttypes.BugPathPos(startLine=14, filePath=None, endCol=6, startCol=3, endLine=14, fileId=1), ttypes.BugPathPos(startLine=15, filePath=None, endCol=3, startCol=3, endLine=15, fileId=1), ttypes.BugPathPos(startLine=16, filePath=None, endCol=1, startCol=1, endLine=16, fileId=1) ] report3_events = [ ttypes.BugPathEvent(startLine=14, filePath=None, endCol=29, startCol=3, msg="Address of stack memory associated" " with local variable 'str'" " is still referred to by the global " "variable 'p' upon returning to the " "caller. This will be a dangling " "reference", endLine=14, fileId=1) ] path, events, _ = store_handler.collect_paths_events( reports[2], file_ids, files) self.assertEqual(path, report3_path) self.assertEqual(events, report3_events)
def test_empty_file(self): """Plist file is empty.""" empty_plist = os.path.join(self.__plist_test_files, 'empty_file') files, reports = plist_parser.parse_plist(empty_plist, None, False) self.assertEquals(files, []) self.assertEquals(reports, [])
def test_collect_path_events(self): """ Test path event collect before store. """ clang50_trunk_plist = os.path.join( self.__plist_test_files, 'clang-5.0-trunk.plist') files, reports = plist_parser.parse_plist(clang50_trunk_plist, None, False) self.assertEqual(len(reports), 3) # Generate dummy file_ids which should come from the database. file_ids = {} for i, file_name in enumerate(files, 1): file_ids[file_name] = i msg = "This test is prepared to handle 3 reports." self.assertEqual(len(reports), 3, msg) report1_path = [ ttypes.BugPathPos(startLine=19, filePath=None, endCol=7, startCol=5, endLine=19, fileId=1), ttypes.BugPathPos(startLine=20, filePath=None, endCol=7, startCol=5, endLine=20, fileId=1), ttypes.BugPathPos(startLine=21, filePath=None, endCol=13, startCol=5, endLine=21, fileId=1), ttypes.BugPathPos(startLine=7, filePath=None, endCol=7, startCol=5, endLine=7, fileId=1), ttypes.BugPathPos(startLine=8, filePath=None, endCol=6, startCol=5, endLine=8, fileId=1), ttypes.BugPathPos(startLine=8, filePath=None, endCol=25, startCol=22, endLine=8, fileId=1), ttypes.BugPathPos(startLine=8, filePath=None, endCol=20, startCol=10, endLine=8, fileId=1), ttypes.BugPathPos(startLine=7, filePath=None, endCol=14, startCol=14, endLine=7, fileId=2) ] report1_events = [ ttypes.BugPathEvent(startLine=20, filePath=None, endCol=12, startCol=5, msg="'base' initialized to 0", endLine=20, fileId=1), ttypes.BugPathEvent(startLine=21, filePath=None, endCol=18, startCol=15, msg="Passing the value 0 via " "1st parameter 'base'", endLine=21, fileId=1), ttypes.BugPathEvent(startLine=21, filePath=None, endCol=19, startCol=5, msg="Calling 'test_func'", endLine=21, fileId=1), ttypes.BugPathEvent(startLine=6, filePath=None, endCol=1, startCol=1, msg="Entered call from 'main'", endLine=6, fileId=1), ttypes.BugPathEvent(startLine=8, filePath=None, endCol=25, startCol=22, msg="Passing the value 0 via " "1st parameter 'num'", endLine=8, fileId=1), ttypes.BugPathEvent(startLine=8, filePath=None, endCol=26, startCol=10, msg="Calling 'generate_id'", endLine=8, fileId=1), ttypes.BugPathEvent(startLine=6, filePath=None, endCol=1, startCol=1, msg="Entered call from 'test_func'", endLine=6, fileId=2), ttypes.BugPathEvent(startLine=7, filePath=None, endCol=17, startCol=12, msg='Division by zero', endLine=7, fileId=2) ] path1, events1, _ = store_handler.collect_paths_events(reports[0], file_ids, files) self.assertEqual(path1, report1_path) self.assertEqual(events1, report1_events) report2_path = [] report2_events = [ ttypes.BugPathEvent(startLine=8, filePath=None, endCol=26, startCol=10, msg="Value stored to 'id' is ""never read", endLine=8, fileId=1) ] path2, events2, _ = store_handler.collect_paths_events(reports[1], file_ids, files) self.assertEqual(path2, report2_path) self.assertEqual(events2, report2_events) report3_path = [ ttypes.BugPathPos(startLine=14, filePath=None, endCol=6, startCol=3, endLine=14, fileId=1), ttypes.BugPathPos(startLine=15, filePath=None, endCol=3, startCol=3, endLine=15, fileId=1), ttypes.BugPathPos(startLine=16, filePath=None, endCol=1, startCol=1, endLine=16, fileId=1) ] report3_events = [ ttypes.BugPathEvent(startLine=14, filePath=None, endCol=29, startCol=3, msg="Address of stack memory associated" " with local variable 'str'" " is still referred to by the global " "variable 'p' upon returning to the " "caller. This will be a dangling " "reference", endLine=14, fileId=1) ] path, events, _ = store_handler.collect_paths_events(reports[2], file_ids, files) self.assertEqual(path, report3_path) self.assertEqual(events, report3_events)