def collect_file_hashes_from_plist(plist_file):
"""
Collects file content hashes and last modification times of files which
can be found in the given plist file.
:returns List of file paths which are in the processed plist file but
missing from the user's disk.
"""
missing_files = []
try:
files, _ = plist_parser.parse_plist(plist_file)
for f in files:
if not os.path.isfile(f):
missing_files.append(f)
missing_source_files.add(f)
continue
content_hash = util.get_file_content_hash(f)
hash_to_file[content_hash] = f
file_to_hash[f] = content_hash
file_to_mtime[f] = util.get_last_mod_time(f)
return missing_files
except Exception as ex:
LOG.error('Parsing the plist failed: ' + str(ex))
def test_clang50_trunk_plist(self):
"""
Check plist generated by clang 5.0 trunk checker name and report hash
should be in the plist file.
"""
clang50_trunk_plist = os.path.join(
self.__plist_test_files, 'clang-5.0-trunk.plist')
files, reports = plist_parser.parse_plist(clang50_trunk_plist, None,
False)
self.assertEquals(files, self.__found_file_names)
self.assertEquals(len(reports), 3)
valid_checker_names = []
valid_checker_names.extend(self.__found_checker_names)
for report in reports:
checker_name = report.main['check_name']
# Checker name should be in the plist file.
self.assertNotEqual(checker_name, 'NOT FOUND')
self.assertIn(checker_name, valid_checker_names)
if checker_name == 'core.DivideZero':
# Test data is still valid for this version.
self.assertEqual(report.main,
div_zero_skel_name_hash)
if checker_name == 'core.StackAddressEscape':
self.assertEqual(report.main,
stack_addr_skel_name_hash_after_v40)
def test_no_bug_file(self):
"""There was no bug in the checked file."""
no_bug_plist = os.path.join(
self.__plist_test_files, 'clang-3.7-noerror.plist')
files, reports = plist_parser.parse_plist(no_bug_plist, None, False)
self.assertEquals(files, [])
self.assertEquals(reports, [])
def get_report_dir_results(reportdir):
all_reports = []
processed_path_hashes = set()
for filename in os.listdir(reportdir):
if filename.endswith(".plist"):
file_path = os.path.join(reportdir, filename)
LOG.debug("Parsing:" + file_path)
try:
files, reports = plist_parser.parse_plist(file_path)
for report in reports:
path_hash = get_report_path_hash(report, files)
if path_hash in processed_path_hashes:
LOG.debug("Not showing report because it is a "
"deduplication of an already processed "
"report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(report)
continue
if skip_report_dir_result(report):
continue
processed_path_hashes.add(path_hash)
report.main['location']['file_name'] = \
files[int(report.main['location']['file'])]
all_reports.append(report)
except Exception as ex:
LOG.error('The generated plist is not valid!')
LOG.error(ex)
return all_reports
def test_clang38_trunk_plist(self):
"""
Check plist generated by clang 3.8 trunk checker name and bug hash
should be in the plist file.
"""
clang38_plist = os.path.join(self.__plist_test_files,
'clang-3.8-trunk.plist')
files, bugs = plist_parser.parse_plist(clang38_plist)
self.assertEquals(files, self.__found_file_names)
self.assertEquals(len(bugs), 3)
valid_checker_names = []
valid_checker_names.extend(self.__found_checker_names)
for bug in bugs:
# Checker name should be in the plist file.
self.assertNotEqual(bug.checker_name, 'NOT FOUND')
self.assertIn(bug.checker_name, valid_checker_names)
if bug.checker_name == 'core.DivideZero':
self.__core_div_bug_event(bug.get_last_event())
self.assertEquals(bug.hash_value,
'79e31a6ba028f0b7d9779faf4a6cb9cf')
if bug.checker_name == 'core.StackAddressEscape':
self.__core_stack_addr_esc_event(bug.get_last_event())
self.assertEquals(bug.hash_value,
'f7b5072d428e890f2d309217f3ead16f')
if bug.checker_name == 'deadcode.DeadStores':
self.assertEquals(bug.hash_value,
'8714f42d8328bc78d5d7bff6ced918cc')
def test_clang37_plist(self):
"""
Check plist generated by clang 3.7 checker name should be in the plist
file.
"""
clang37_plist = os.path.join(self.__plist_test_files,
'clang-3.7.plist')
files, bugs = plist_parser.parse_plist(clang37_plist)
self.assertEquals(files, self.__found_file_names)
self.assertEquals(len(bugs), 3)
valid_checker_names = []
valid_checker_names.extend(self.__found_checker_names)
for bug in bugs:
# Checker name should be in the plist file.
self.assertNotEqual(bug.checker_name, 'NOT FOUND')
self.assertIn(bug.checker_name, valid_checker_names)
if bug.checker_name == 'core.DivideZero':
self.__core_div_bug_event(bug.get_last_event())
if bug.checker_name == 'core.StackAddressEscape':
self.__core_stack_addr_esc_event(bug.get_last_event())
self.assertNotEquals(bug.hash_value, '')
def test_report_path_hash_generation(self):
"""
Test report path hash generation.
"""
clang50_trunk_plist = os.path.join(self.__plist_test_files,
'clang-5.0-trunk.plist')
files, reports = plist_parser.parse_plist(clang50_trunk_plist, None,
False)
self.assertEqual(len(reports), 3)
# Generate dummy file_ids which should come from the database.
file_ids = {}
for i, file_name in enumerate(files, 1):
file_ids[file_name] = i
msg = "This test is prepared to handle 3 reports."
self.assertEqual(len(reports), 3, msg)
report_hash_to_path_hash = {
'79e31a6ba028f0b7d9779faf4a6cb9cf':
'c473c1a55df72ea4c6e055e18370ac65',
'8714f42d8328bc78d5d7bff6ced918cc':
'94f2a6eee8af6462a810218dff35056a',
'a6d3464f8aab9eb31a8ea7e167e84322':
'11f410136724cf43c63526841007897e'
}
for report in reports:
path_hash = get_report_path_hash(report, files)
bug_hash = report.main['issue_hash_content_of_line_in_context']
self.assertEqual(path_hash, report_hash_to_path_hash[bug_hash])
def handle_results(self, client=None):
plist = self.analyzer_result_file
report_stats = {}
try:
files, reports = plist_parser.parse_plist(plist)
except Exception as ex:
LOG.error('The generated plist is not valid!')
LOG.error(ex)
return 1
err_code = self.analyzer_returncode
if err_code == 0:
try:
# No lock when consuming plist.
if self.__lock:
self.__lock.acquire()
self.__print_bugs(reports, files, report_stats)
finally:
if self.__lock:
self.__lock.release()
else:
self.__output.write('Analyzing %s with %s failed.\n' %
(os.path.basename(self.analyzed_source_file),
self.buildaction.analyzer_type))
return report_stats
def handle_results(self, client=None):
"""
Send the plist content to the database.
Server API calls should be used in one connection.
- addBuildAction
- addReport
- needFileContent
- addFileContent
- finishBuildAction
"""
if not client:
LOG.error("Client needs to be set to store the "
"results to the database")
return
LOG.debug('Storing original build and analyzer command '
'to the database.')
_, source_file_name = os.path.split(self.analyzed_source_file)
if LoggerFactory.get_log_level() == logger.DEBUG:
analyzer_cmd = ' '.join(self.analyzer_cmd)
else:
analyzer_cmd = ''
build_cmd_hash = self.buildaction.original_command_hash
analysis_id = \
client.addBuildAction(self.__run_id,
build_cmd_hash,
analyzer_cmd,
self.buildaction.analyzer_type,
source_file_name)
plist_file = self.analyzer_result_file
try:
files, reports = plist_parser.parse_plist(plist_file)
except Exception as ex:
LOG.debug(str(ex))
msg = 'Parsing the generated result file failed.'
LOG.error(msg + ' ' + plist_file)
client.finishBuildAction(analysis_id, msg)
return 1
try:
self.__store_bugs(files, reports, client, analysis_id)
except Exception as ex:
LOG.error("Failed to store results")
traceback.print_stack()
LOG.error(ex)
return 1
finally:
LOG.debug("Finishing buildaction")
client.finishBuildAction(analysis_id, self.analyzer_stderr)
LOG.debug("Finishing buildaction done.")
def test_clang36_plist(self):
"""
Check plist generated by clang 3.6 checker name is not in the plist
file plist parser tries to find out the name and generate a report
hash.
"""
clang36_plist = os.path.join(self.__plist_test_files,
'clang-3.6.plist')
files, reports = plist_parser.parse_plist(clang36_plist)
self.assertEquals(files, self.__found_file_names)
self.assertEquals(len(reports), 3)
self.missing_checker_name_and_hash(reports)
def test_clang37_plist(self):
"""
Check plist generated by clang 3.7 checker name should be in the plist
file generating a report hash is still needed.
"""
clang37_plist = os.path.join(
self.__plist_test_files, 'clang-3.7.plist')
files, reports = plist_parser.parse_plist(clang37_plist, None, False)
self.assertEquals(files, self.__found_file_names)
self.assertEquals(len(reports), 3)
self.missing_hash(reports)
def collect_file_hashes_from_plist(plist_file):
try:
files, _ = plist_parser.parse_plist(plist_file)
for f in files:
if not os.path.isfile(f):
return False
content_hash = util.get_file_content_hash(f)
hash_to_file[content_hash] = f
file_to_hash[f] = content_hash
file_to_mtime[f] = util.get_last_mod_time(f)
return True
except Exception as ex:
LOG.error('Parsing the plist failed: ' + str(ex))
def handle_results(self):
"""
Send the plist content to the database.
Server API calls should be used in one connection.
- addBuildAction
- addReport
- needFileContent
- addFileContent
- finishBuildAction
"""
with client.get_connection() as connection:
LOG.debug('Storing original build and analyzer command '
'to the database.')
_, source_file_name = os.path.split(self.analyzed_source_file)
if LoggerFactory.get_log_level() == logger.DEBUG:
analyzer_cmd = ' '.join(self.analyzer_cmd)
else:
analyzer_cmd = ''
build_cmd_hash = self.buildaction.original_command_hash
analysis_id = \
connection.add_build_action(self.__run_id,
build_cmd_hash,
analyzer_cmd,
self.buildaction.analyzer_type,
source_file_name)
assert self.analyzer_returncode == 0
plist_file = self.analyzer_result_file
try:
files, reports = plist_parser.parse_plist(plist_file)
except Exception as ex:
LOG.debug(str(ex))
msg = 'Parsing the generated result file failed.'
LOG.error(msg + ' ' + plist_file)
connection.finish_build_action(analysis_id, msg)
return 1
self.__store_bugs(files, reports, connection, analysis_id)
connection.finish_build_action(analysis_id, self.analyzer_stderr)
def get_report_dir_results(reportdir):
all_reports = []
for filename in os.listdir(reportdir):
if filename.endswith(".plist"):
file_path = os.path.join(reportdir, filename)
LOG.debug("Parsing:" + file_path)
try:
files, reports = plist_parser.parse_plist(file_path)
for report in reports:
report.main['location']['file_name'] = \
files[int(report.main['location']['file'])]
all_reports.extend(reports)
except Exception as ex:
LOG.error('The generated plist is not valid!')
LOG.error(ex)
return all_reports
def collect_file_hashes_from_plist(plist_file):
try:
files, _ = plist_parser.parse_plist(plist_file)
for f in files:
if not os.path.isfile(f):
return False
with open(f) as content:
hasher = sha256()
hasher.update(content.read())
content_hash = hasher.hexdigest()
hash_to_file[content_hash] = f
file_to_hash[f] = content_hash
return True
except Exception as ex:
LOG.error('Parsing the plist failed: ' + str(ex))
def test_clang35_plist(self):
"""
Check plist generated by clang 3.5 checker name is not in the plist
file plist parser tries to find out the name.
"""
clang35_plist = os.path.join(self.__plist_test_files,
'clang-3.5.plist')
files, bugs = plist_parser.parse_plist(clang35_plist)
self.assertEquals(files, self.__found_file_names)
self.assertEquals(len(bugs), 3)
valid_checker_names = []
valid_checker_names.extend(self.__found_checker_names)
valid_checker_names.extend(self.__not_found_checker_names)
for bug in bugs:
self.assertIn(bug.checker_name, valid_checker_names)
if bug.checker_name == 'core.DivideZero':
self.__core_div_bug_event(bug.get_last_event())
if bug.checker_name == 'NOT FOUND':
self.__core_stack_addr_esc_event(bug.get_last_event())
self.assertIn(bug.hash_value, self.__framework_generated_hashes)
def handle_results(self):
plist = self.analyzer_result_file
try:
_, bugs = plist_parser.parse_plist(plist)
except Exception as ex:
LOG.error('The generated plist is not valid!')
LOG.error(ex)
return 1
err_code = self.analyzer_returncode
if err_code == 0:
try:
# No lock when consuming plist.
self.__lock.acquire() if self.__lock else None
self.__print_bugs(bugs)
finally:
self.__lock.release() if self.__lock else None
else:
self.__output.write('Analyzing %s with %s failed.\n' %
(ntpath.basename(self.analyzed_source_file),
self.buildaction.analyzer_type))
return err_code
def test_empty_file(self):
"""Plist file is empty."""
empty_plist = os.path.join(self.__plist_test_files, 'empty_file')
files, bugs = plist_parser.parse_plist(empty_plist)
self.assertEquals(files, [])
self.assertEquals(bugs, [])
def test_empty_file(self):
"""Plist file is empty."""
empty_plist = os.path.join(self.__plist_test_files, 'empty_file')
files, reports = plist_parser.parse_plist(empty_plist, None, False)
self.assertEquals(files, [])
self.assertEquals(reports, [])
def test_collect_path_events(self):
"""
Test path event collect before store.
"""
clang50_trunk_plist = os.path.join(self.__plist_test_files,
'clang-5.0-trunk.plist')
files, reports = plist_parser.parse_plist(clang50_trunk_plist, None,
False)
self.assertEqual(len(reports), 3)
# Generate dummy file_ids which should come from the database.
file_ids = {}
for i, file_name in enumerate(files, 1):
file_ids[file_name] = i
msg = "This test is prepared to handle 3 reports."
self.assertEqual(len(reports), 3, msg)
report1_path = [
ttypes.BugPathPos(startLine=19,
filePath=None,
endCol=7,
startCol=5,
endLine=19,
fileId=1),
ttypes.BugPathPos(startLine=20,
filePath=None,
endCol=7,
startCol=5,
endLine=20,
fileId=1),
ttypes.BugPathPos(startLine=21,
filePath=None,
endCol=13,
startCol=5,
endLine=21,
fileId=1),
ttypes.BugPathPos(startLine=7,
filePath=None,
endCol=7,
startCol=5,
endLine=7,
fileId=1),
ttypes.BugPathPos(startLine=8,
filePath=None,
endCol=6,
startCol=5,
endLine=8,
fileId=1),
ttypes.BugPathPos(startLine=8,
filePath=None,
endCol=25,
startCol=22,
endLine=8,
fileId=1),
ttypes.BugPathPos(startLine=8,
filePath=None,
endCol=20,
startCol=10,
endLine=8,
fileId=1),
ttypes.BugPathPos(startLine=7,
filePath=None,
endCol=14,
startCol=14,
endLine=7,
fileId=2)
]
report1_events = [
ttypes.BugPathEvent(startLine=20,
filePath=None,
endCol=12,
startCol=5,
msg="'base' initialized to 0",
endLine=20,
fileId=1),
ttypes.BugPathEvent(startLine=21,
filePath=None,
endCol=18,
startCol=15,
msg="Passing the value 0 via "
"1st parameter 'base'",
endLine=21,
fileId=1),
ttypes.BugPathEvent(startLine=21,
filePath=None,
endCol=19,
startCol=5,
msg="Calling 'test_func'",
endLine=21,
fileId=1),
ttypes.BugPathEvent(startLine=6,
filePath=None,
endCol=1,
startCol=1,
msg="Entered call from 'main'",
endLine=6,
fileId=1),
ttypes.BugPathEvent(startLine=8,
filePath=None,
endCol=25,
startCol=22,
msg="Passing the value 0 via "
"1st parameter 'num'",
endLine=8,
fileId=1),
ttypes.BugPathEvent(startLine=8,
filePath=None,
endCol=26,
startCol=10,
msg="Calling 'generate_id'",
endLine=8,
fileId=1),
ttypes.BugPathEvent(startLine=6,
filePath=None,
endCol=1,
startCol=1,
msg="Entered call from 'test_func'",
endLine=6,
fileId=2),
ttypes.BugPathEvent(startLine=7,
filePath=None,
endCol=17,
startCol=12,
msg='Division by zero',
endLine=7,
fileId=2)
]
path1, events1, _ = store_handler.collect_paths_events(
reports[0], file_ids, files)
self.assertEqual(path1, report1_path)
self.assertEqual(events1, report1_events)
report2_path = []
report2_events = [
ttypes.BugPathEvent(startLine=8,
filePath=None,
endCol=26,
startCol=10,
msg="Value stored to 'id' is "
"never read",
endLine=8,
fileId=1)
]
path2, events2, _ = store_handler.collect_paths_events(
reports[1], file_ids, files)
self.assertEqual(path2, report2_path)
self.assertEqual(events2, report2_events)
report3_path = [
ttypes.BugPathPos(startLine=14,
filePath=None,
endCol=6,
startCol=3,
endLine=14,
fileId=1),
ttypes.BugPathPos(startLine=15,
filePath=None,
endCol=3,
startCol=3,
endLine=15,
fileId=1),
ttypes.BugPathPos(startLine=16,
filePath=None,
endCol=1,
startCol=1,
endLine=16,
fileId=1)
]
report3_events = [
ttypes.BugPathEvent(startLine=14,
filePath=None,
endCol=29,
startCol=3,
msg="Address of stack memory associated"
" with local variable 'str'"
" is still referred to by the global "
"variable 'p' upon returning to the "
"caller. This will be a dangling "
"reference",
endLine=14,
fileId=1)
]
path, events, _ = store_handler.collect_paths_events(
reports[2], file_ids, files)
self.assertEqual(path, report3_path)
self.assertEqual(events, report3_events)
