def collect_file_hashes_from_plist(plist_file): """ Collects file content hashes and last modification times of files which can be found in the given plist file. :returns List of file paths which are in the processed plist file but missing from the user's disk. """ missing_files = [] try: files, _ = plist_parser.parse_plist(plist_file) for f in files: if not os.path.isfile(f): missing_files.append(f) missing_source_files.add(f) continue content_hash = util.get_file_content_hash(f) hash_to_file[content_hash] = f file_to_hash[f] = content_hash file_to_mtime[f] = util.get_last_mod_time(f) return missing_files except Exception as ex: LOG.error('Parsing the plist failed: ' + str(ex))
def test_clang50_trunk_plist(self): """ Check plist generated by clang 5.0 trunk checker name and report hash should be in the plist file. """ clang50_trunk_plist = os.path.join( self.__plist_test_files, 'clang-5.0-trunk.plist') files, reports = plist_parser.parse_plist(clang50_trunk_plist, None, False) self.assertEquals(files, self.__found_file_names) self.assertEquals(len(reports), 3) valid_checker_names = [] valid_checker_names.extend(self.__found_checker_names) for report in reports: checker_name = report.main['check_name'] # Checker name should be in the plist file. self.assertNotEqual(checker_name, 'NOT FOUND') self.assertIn(checker_name, valid_checker_names) if checker_name == 'core.DivideZero': # Test data is still valid for this version. self.assertEqual(report.main, div_zero_skel_name_hash) if checker_name == 'core.StackAddressEscape': self.assertEqual(report.main, stack_addr_skel_name_hash_after_v40)
def test_no_bug_file(self): """There was no bug in the checked file.""" no_bug_plist = os.path.join( self.__plist_test_files, 'clang-3.7-noerror.plist') files, reports = plist_parser.parse_plist(no_bug_plist, None, False) self.assertEquals(files, []) self.assertEquals(reports, [])
def get_report_dir_results(reportdir): all_reports = [] processed_path_hashes = set() for filename in os.listdir(reportdir): if filename.endswith(".plist"): file_path = os.path.join(reportdir, filename) LOG.debug("Parsing:" + file_path) try: files, reports = plist_parser.parse_plist(file_path) for report in reports: path_hash = get_report_path_hash(report, files) if path_hash in processed_path_hashes: LOG.debug("Not showing report because it is a " "deduplication of an already processed " "report!") LOG.debug("Path hash: %s", path_hash) LOG.debug(report) continue if skip_report_dir_result(report): continue processed_path_hashes.add(path_hash) report.main['location']['file_name'] = \ files[int(report.main['location']['file'])] all_reports.append(report) except Exception as ex: LOG.error('The generated plist is not valid!') LOG.error(ex) return all_reports
def test_clang38_trunk_plist(self): """ Check plist generated by clang 3.8 trunk checker name and bug hash should be in the plist file. """ clang38_plist = os.path.join(self.__plist_test_files, 'clang-3.8-trunk.plist') files, bugs = plist_parser.parse_plist(clang38_plist) self.assertEquals(files, self.__found_file_names) self.assertEquals(len(bugs), 3) valid_checker_names = [] valid_checker_names.extend(self.__found_checker_names) for bug in bugs: # Checker name should be in the plist file. self.assertNotEqual(bug.checker_name, 'NOT FOUND') self.assertIn(bug.checker_name, valid_checker_names) if bug.checker_name == 'core.DivideZero': self.__core_div_bug_event(bug.get_last_event()) self.assertEquals(bug.hash_value, '79e31a6ba028f0b7d9779faf4a6cb9cf') if bug.checker_name == 'core.StackAddressEscape': self.__core_stack_addr_esc_event(bug.get_last_event()) self.assertEquals(bug.hash_value, 'f7b5072d428e890f2d309217f3ead16f') if bug.checker_name == 'deadcode.DeadStores': self.assertEquals(bug.hash_value, '8714f42d8328bc78d5d7bff6ced918cc')
def test_clang37_plist(self): """ Check plist generated by clang 3.7 checker name should be in the plist file. """ clang37_plist = os.path.join(self.__plist_test_files, 'clang-3.7.plist') files, bugs = plist_parser.parse_plist(clang37_plist) self.assertEquals(files, self.__found_file_names) self.assertEquals(len(bugs), 3) valid_checker_names = [] valid_checker_names.extend(self.__found_checker_names) for bug in bugs: # Checker name should be in the plist file. self.assertNotEqual(bug.checker_name, 'NOT FOUND') self.assertIn(bug.checker_name, valid_checker_names) if bug.checker_name == 'core.DivideZero': self.__core_div_bug_event(bug.get_last_event()) if bug.checker_name == 'core.StackAddressEscape': self.__core_stack_addr_esc_event(bug.get_last_event()) self.assertNotEquals(bug.hash_value, '')
def test_report_path_hash_generation(self): """ Test report path hash generation. """ clang50_trunk_plist = os.path.join(self.__plist_test_files, 'clang-5.0-trunk.plist') files, reports = plist_parser.parse_plist(clang50_trunk_plist, None, False) self.assertEqual(len(reports), 3) # Generate dummy file_ids which should come from the database. file_ids = {} for i, file_name in enumerate(files, 1): file_ids[file_name] = i msg = "This test is prepared to handle 3 reports." self.assertEqual(len(reports), 3, msg) report_hash_to_path_hash = { '79e31a6ba028f0b7d9779faf4a6cb9cf': 'c473c1a55df72ea4c6e055e18370ac65', '8714f42d8328bc78d5d7bff6ced918cc': '94f2a6eee8af6462a810218dff35056a', 'a6d3464f8aab9eb31a8ea7e167e84322': '11f410136724cf43c63526841007897e' } for report in reports: path_hash = get_report_path_hash(report, files) bug_hash = report.main['issue_hash_content_of_line_in_context'] self.assertEqual(path_hash, report_hash_to_path_hash[bug_hash])
def handle_results(self, client=None): plist = self.analyzer_result_file report_stats = {} try: files, reports = plist_parser.parse_plist(plist) except Exception as ex: LOG.error('The generated plist is not valid!') LOG.error(ex) return 1 err_code = self.analyzer_returncode if err_code == 0: try: # No lock when consuming plist. if self.__lock: self.__lock.acquire() self.__print_bugs(reports, files, report_stats) finally: if self.__lock: self.__lock.release() else: self.__output.write('Analyzing %s with %s failed.\n' % (os.path.basename(self.analyzed_source_file), self.buildaction.analyzer_type)) return report_stats
def handle_results(self, client=None): """ Send the plist content to the database. Server API calls should be used in one connection. - addBuildAction - addReport - needFileContent - addFileContent - finishBuildAction """ if not client: LOG.error("Client needs to be set to store the " "results to the database") return LOG.debug('Storing original build and analyzer command ' 'to the database.') _, source_file_name = os.path.split(self.analyzed_source_file) if LoggerFactory.get_log_level() == logger.DEBUG: analyzer_cmd = ' '.join(self.analyzer_cmd) else: analyzer_cmd = '' build_cmd_hash = self.buildaction.original_command_hash analysis_id = \ client.addBuildAction(self.__run_id, build_cmd_hash, analyzer_cmd, self.buildaction.analyzer_type, source_file_name) plist_file = self.analyzer_result_file try: files, reports = plist_parser.parse_plist(plist_file) except Exception as ex: LOG.debug(str(ex)) msg = 'Parsing the generated result file failed.' LOG.error(msg + ' ' + plist_file) client.finishBuildAction(analysis_id, msg) return 1 try: self.__store_bugs(files, reports, client, analysis_id) except Exception as ex: LOG.error("Failed to store results") traceback.print_stack() LOG.error(ex) return 1 finally: LOG.debug("Finishing buildaction") client.finishBuildAction(analysis_id, self.analyzer_stderr) LOG.debug("Finishing buildaction done.")
def test_clang36_plist(self): """ Check plist generated by clang 3.6 checker name is not in the plist file plist parser tries to find out the name and generate a report hash. """ clang36_plist = os.path.join(self.__plist_test_files, 'clang-3.6.plist') files, reports = plist_parser.parse_plist(clang36_plist) self.assertEquals(files, self.__found_file_names) self.assertEquals(len(reports), 3) self.missing_checker_name_and_hash(reports)
def test_clang37_plist(self): """ Check plist generated by clang 3.7 checker name should be in the plist file generating a report hash is still needed. """ clang37_plist = os.path.join( self.__plist_test_files, 'clang-3.7.plist') files, reports = plist_parser.parse_plist(clang37_plist, None, False) self.assertEquals(files, self.__found_file_names) self.assertEquals(len(reports), 3) self.missing_hash(reports)
def collect_file_hashes_from_plist(plist_file): try: files, _ = plist_parser.parse_plist(plist_file) for f in files: if not os.path.isfile(f): return False content_hash = util.get_file_content_hash(f) hash_to_file[content_hash] = f file_to_hash[f] = content_hash file_to_mtime[f] = util.get_last_mod_time(f) return True except Exception as ex: LOG.error('Parsing the plist failed: ' + str(ex))
def handle_results(self): """ Send the plist content to the database. Server API calls should be used in one connection. - addBuildAction - addReport - needFileContent - addFileContent - finishBuildAction """ with client.get_connection() as connection: LOG.debug('Storing original build and analyzer command ' 'to the database.') _, source_file_name = os.path.split(self.analyzed_source_file) if LoggerFactory.get_log_level() == logger.DEBUG: analyzer_cmd = ' '.join(self.analyzer_cmd) else: analyzer_cmd = '' build_cmd_hash = self.buildaction.original_command_hash analysis_id = \ connection.add_build_action(self.__run_id, build_cmd_hash, analyzer_cmd, self.buildaction.analyzer_type, source_file_name) assert self.analyzer_returncode == 0 plist_file = self.analyzer_result_file try: files, reports = plist_parser.parse_plist(plist_file) except Exception as ex: LOG.debug(str(ex)) msg = 'Parsing the generated result file failed.' LOG.error(msg + ' ' + plist_file) connection.finish_build_action(analysis_id, msg) return 1 self.__store_bugs(files, reports, connection, analysis_id) connection.finish_build_action(analysis_id, self.analyzer_stderr)
def get_report_dir_results(reportdir): all_reports = [] for filename in os.listdir(reportdir): if filename.endswith(".plist"): file_path = os.path.join(reportdir, filename) LOG.debug("Parsing:" + file_path) try: files, reports = plist_parser.parse_plist(file_path) for report in reports: report.main['location']['file_name'] = \ files[int(report.main['location']['file'])] all_reports.extend(reports) except Exception as ex: LOG.error('The generated plist is not valid!') LOG.error(ex) return all_reports
def collect_file_hashes_from_plist(plist_file): try: files, _ = plist_parser.parse_plist(plist_file) for f in files: if not os.path.isfile(f): return False with open(f) as content: hasher = sha256() hasher.update(content.read()) content_hash = hasher.hexdigest() hash_to_file[content_hash] = f file_to_hash[f] = content_hash return True except Exception as ex: LOG.error('Parsing the plist failed: ' + str(ex))
def test_clang35_plist(self): """ Check plist generated by clang 3.5 checker name is not in the plist file plist parser tries to find out the name. """ clang35_plist = os.path.join(self.__plist_test_files, 'clang-3.5.plist') files, bugs = plist_parser.parse_plist(clang35_plist) self.assertEquals(files, self.__found_file_names) self.assertEquals(len(bugs), 3) valid_checker_names = [] valid_checker_names.extend(self.__found_checker_names) valid_checker_names.extend(self.__not_found_checker_names) for bug in bugs: self.assertIn(bug.checker_name, valid_checker_names) if bug.checker_name == 'core.DivideZero': self.__core_div_bug_event(bug.get_last_event()) if bug.checker_name == 'NOT FOUND': self.__core_stack_addr_esc_event(bug.get_last_event()) self.assertIn(bug.hash_value, self.__framework_generated_hashes)
def handle_results(self): plist = self.analyzer_result_file try: _, bugs = plist_parser.parse_plist(plist) except Exception as ex: LOG.error('The generated plist is not valid!') LOG.error(ex) return 1 err_code = self.analyzer_returncode if err_code == 0: try: # No lock when consuming plist. self.__lock.acquire() if self.__lock else None self.__print_bugs(bugs) finally: self.__lock.release() if self.__lock else None else: self.__output.write('Analyzing %s with %s failed.\n' % (ntpath.basename(self.analyzed_source_file), self.buildaction.analyzer_type)) return err_code
def test_empty_file(self): """Plist file is empty.""" empty_plist = os.path.join(self.__plist_test_files, 'empty_file') files, bugs = plist_parser.parse_plist(empty_plist) self.assertEquals(files, []) self.assertEquals(bugs, [])
def test_empty_file(self): """Plist file is empty.""" empty_plist = os.path.join(self.__plist_test_files, 'empty_file') files, reports = plist_parser.parse_plist(empty_plist, None, False) self.assertEquals(files, []) self.assertEquals(reports, [])
def test_collect_path_events(self): """ Test path event collect before store. """ clang50_trunk_plist = os.path.join(self.__plist_test_files, 'clang-5.0-trunk.plist') files, reports = plist_parser.parse_plist(clang50_trunk_plist, None, False) self.assertEqual(len(reports), 3) # Generate dummy file_ids which should come from the database. file_ids = {} for i, file_name in enumerate(files, 1): file_ids[file_name] = i msg = "This test is prepared to handle 3 reports." self.assertEqual(len(reports), 3, msg) report1_path = [ ttypes.BugPathPos(startLine=19, filePath=None, endCol=7, startCol=5, endLine=19, fileId=1), ttypes.BugPathPos(startLine=20, filePath=None, endCol=7, startCol=5, endLine=20, fileId=1), ttypes.BugPathPos(startLine=21, filePath=None, endCol=13, startCol=5, endLine=21, fileId=1), ttypes.BugPathPos(startLine=7, filePath=None, endCol=7, startCol=5, endLine=7, fileId=1), ttypes.BugPathPos(startLine=8, filePath=None, endCol=6, startCol=5, endLine=8, fileId=1), ttypes.BugPathPos(startLine=8, filePath=None, endCol=25, startCol=22, endLine=8, fileId=1), ttypes.BugPathPos(startLine=8, filePath=None, endCol=20, startCol=10, endLine=8, fileId=1), ttypes.BugPathPos(startLine=7, filePath=None, endCol=14, startCol=14, endLine=7, fileId=2) ] report1_events = [ ttypes.BugPathEvent(startLine=20, filePath=None, endCol=12, startCol=5, msg="'base' initialized to 0", endLine=20, fileId=1), ttypes.BugPathEvent(startLine=21, filePath=None, endCol=18, startCol=15, msg="Passing the value 0 via " "1st parameter 'base'", endLine=21, fileId=1), ttypes.BugPathEvent(startLine=21, filePath=None, endCol=19, startCol=5, msg="Calling 'test_func'", endLine=21, fileId=1), ttypes.BugPathEvent(startLine=6, filePath=None, endCol=1, startCol=1, msg="Entered call from 'main'", endLine=6, fileId=1), ttypes.BugPathEvent(startLine=8, filePath=None, endCol=25, startCol=22, msg="Passing the value 0 via " "1st parameter 'num'", endLine=8, fileId=1), ttypes.BugPathEvent(startLine=8, filePath=None, endCol=26, startCol=10, msg="Calling 'generate_id'", endLine=8, fileId=1), ttypes.BugPathEvent(startLine=6, filePath=None, endCol=1, startCol=1, msg="Entered call from 'test_func'", endLine=6, fileId=2), ttypes.BugPathEvent(startLine=7, filePath=None, endCol=17, startCol=12, msg='Division by zero', endLine=7, fileId=2) ] path1, events1, _ = store_handler.collect_paths_events( reports[0], file_ids, files) self.assertEqual(path1, report1_path) self.assertEqual(events1, report1_events) report2_path = [] report2_events = [ ttypes.BugPathEvent(startLine=8, filePath=None, endCol=26, startCol=10, msg="Value stored to 'id' is " "never read", endLine=8, fileId=1) ] path2, events2, _ = store_handler.collect_paths_events( reports[1], file_ids, files) self.assertEqual(path2, report2_path) self.assertEqual(events2, report2_events) report3_path = [ ttypes.BugPathPos(startLine=14, filePath=None, endCol=6, startCol=3, endLine=14, fileId=1), ttypes.BugPathPos(startLine=15, filePath=None, endCol=3, startCol=3, endLine=15, fileId=1), ttypes.BugPathPos(startLine=16, filePath=None, endCol=1, startCol=1, endLine=16, fileId=1) ] report3_events = [ ttypes.BugPathEvent(startLine=14, filePath=None, endCol=29, startCol=3, msg="Address of stack memory associated" " with local variable 'str'" " is still referred to by the global " "variable 'p' upon returning to the " "caller. This will be a dangling " "reference", endLine=14, fileId=1) ] path, events, _ = store_handler.collect_paths_events( reports[2], file_ids, files) self.assertEqual(path, report3_path) self.assertEqual(events, report3_events)