def test_clang50_trunk_plist(self):
"""
Check plist generated by clang 5.0 trunk checker name and report hash
should be in the plist file.
"""
clang50_trunk_plist = os.path.join(self.__plist_test_files,
'clang-5.0-trunk.plist')
files, reports = plist_parser.parse_plist(clang50_trunk_plist, None,
False)
self.assertEquals(files, self.__found_file_names)
self.assertEquals(len(reports), 3)
valid_checker_names = []
valid_checker_names.extend(self.__found_checker_names)
for report in reports:
checker_name = report.main['check_name']
# Checker name should be in the plist file.
self.assertNotEqual(checker_name, 'NOT FOUND')
self.assertIn(checker_name, valid_checker_names)
if checker_name == 'core.DivideZero':
# Test data is still valid for this version.
self.assertEqual(report.main, div_zero_skel_name_hash)
if checker_name == 'core.StackAddressEscape':
self.assertEqual(report.main,
stack_addr_skel_name_hash_after_v40)
def get_report_dir_results(reportdir):
all_reports = []
processed_path_hashes = set()
for filename in os.listdir(reportdir):
if filename.endswith(".plist"):
file_path = os.path.join(reportdir, filename)
LOG.debug("Parsing: %s", file_path)
try:
files, reports = plist_parser.parse_plist(file_path)
for report in reports:
path_hash = get_report_path_hash(report, files)
if path_hash in processed_path_hashes:
LOG.debug("Not showing report because it is a "
"deduplication of an already processed "
"report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(report)
continue
if skip_report_dir_result(report):
continue
processed_path_hashes.add(path_hash)
report.main['location']['file_name'] = \
files[int(report.main['location']['file'])]
all_reports.append(report)
except Exception as ex:
LOG.error('The generated plist is not valid!')
LOG.error(ex)
return all_reports
def test_no_bug_file(self):
"""There was no bug in the checked file."""
no_bug_plist = os.path.join(self.__plist_test_files,
'clang-3.7-noerror.plist')
files, reports = plist_parser.parse_plist(no_bug_plist, None, False)
self.assertEquals(files, [])
self.assertEquals(reports, [])
def get_report_dir_results(reportdir):
all_reports = []
processed_path_hashes = set()
for filename in os.listdir(reportdir):
if filename.endswith(".plist"):
file_path = os.path.join(reportdir, filename)
LOG.debug("Parsing: %s", file_path)
try:
files, reports = plist_parser.parse_plist(file_path)
for report in reports:
path_hash = get_report_path_hash(report, files)
if path_hash in processed_path_hashes:
LOG.debug("Not showing report because it is a "
"deduplication of an already processed "
"report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(report)
continue
if skip_report_dir_result(report):
continue
processed_path_hashes.add(path_hash)
report.main['location']['file_name'] = \
files[int(report.main['location']['file'])]
all_reports.append(report)
except Exception as ex:
LOG.error('The generated plist is not valid!')
LOG.error(ex)
return all_reports
def test_report_path_hash_generation(self):
"""
Test report path hash generation.
"""
clang50_trunk_plist = os.path.join(self.__plist_test_files,
'clang-5.0-trunk.plist')
files, reports = plist_parser.parse_plist(clang50_trunk_plist, None,
False)
self.assertEqual(len(reports), 3)
# Generate dummy file_ids which should come from the database.
file_ids = {}
for i, file_name in enumerate(files, 1):
file_ids[file_name] = i
msg = "This test is prepared to handle 3 reports."
self.assertEqual(len(reports), 3, msg)
report_hash_to_path_hash = {
'79e31a6ba028f0b7d9779faf4a6cb9cf':
'c473c1a55df72ea4c6e055e18370ac65',
'8714f42d8328bc78d5d7bff6ced918cc':
'94f2a6eee8af6462a810218dff35056a',
'a6d3464f8aab9eb31a8ea7e167e84322':
'11f410136724cf43c63526841007897e'
}
for report in reports:
path_hash = get_report_path_hash(report, files)
bug_hash = report.main['issue_hash_content_of_line_in_context']
self.assertEqual(path_hash, report_hash_to_path_hash[bug_hash])
def collect_file_hashes_from_plist(plist_file):
"""
Collects file content hashes and last modification times of files which
can be found in the given plist file.
:returns List of file paths which are in the processed plist file but
missing from the user's disk.
"""
missing_files = []
try:
files, _ = plist_parser.parse_plist(plist_file)
for f in files:
if not os.path.isfile(f):
missing_files.append(f)
missing_source_files.add(f)
continue
content_hash = util.get_file_content_hash(f)
hash_to_file[content_hash] = f
file_to_hash[f] = content_hash
file_to_mtime[f] = util.get_last_mod_time(f)
return missing_files
except Exception as ex:
LOG.error('Parsing the plist failed: %s', str(ex))
def test_clang37_plist(self):
"""
Check plist generated by clang 3.7 checker name should be in the plist
file generating a report hash is still needed.
"""
clang37_plist = os.path.join(self.__plist_test_files,
'clang-3.7.plist')
files, reports = plist_parser.parse_plist(clang37_plist, None, False)
self.assertEquals(files, self.__found_file_names)
self.assertEquals(len(reports), 3)
self.missing_hash(reports)
def collect_file_hashes_from_plist(plist_file):
"""
Collects file content hashes and last modification times for the
source files which can be found in the given plist file.
:returns List of file paths which are in the processed plist file but
missing from the user's disk and the source file modification times
for the still available source files.
"""
source_file_mod_times = {}
missing_files = []
try:
files, reports = plist_parser.parse_plist(plist_file)
for f in files:
if not os.path.isfile(f):
missing_files.append(f)
missing_source_files.add(f)
continue
content_hash = get_file_content_hash(f)
hash_to_file[content_hash] = f
file_to_hash[f] = content_hash
source_file_mod_times[f] = util.get_last_mod_time(f)
# Get file hashes which contain source code comments.
for report in reports:
last_report_event = report.bug_path[-1]
file_path = files[last_report_event['location']['file']]
if not os.path.isfile(file_path):
continue
file_hash = file_to_hash[file_path]
if file_hash in file_hash_with_review_status:
continue
report_line = last_report_event['location']['line']
sc_handler = SourceCodeCommentHandler(file_path)
if sc_handler.has_source_line_comments(report_line):
file_hash_with_review_status.add(file_hash)
return missing_files, source_file_mod_times
except Exception as ex:
LOG.error('Parsing the plist failed: %s', str(ex))
def collect_file_hashes_from_plist(plist_file):
"""
Collects file content hashes and last modification times of files which
can be found in the given plist file.
:returns List of file paths which are in the processed plist file but
missing from the user's disk.
"""
missing_files = []
try:
files, reports = plist_parser.parse_plist(plist_file)
for f in files:
if not os.path.isfile(f):
missing_files.append(f)
missing_source_files.add(f)
continue
content_hash = util.get_file_content_hash(f)
hash_to_file[content_hash] = f
file_to_hash[f] = content_hash
file_to_mtime[f] = util.get_last_mod_time(f)
# Get file hashes which contain source code comments.
for report in reports:
last_report_event = report.bug_path[-1]
file_path = files[last_report_event['location']['file']]
if not os.path.isfile(file_path):
continue
file_hash = file_to_hash[file_path]
if file_hash in file_hash_with_review_status:
continue
report_line = last_report_event['location']['line']
sc_handler = SourceCodeCommentHandler(file_path)
if sc_handler.has_source_line_comments(report_line):
file_hash_with_review_status.add(file_hash)
return missing_files
except Exception as ex:
LOG.error('Parsing the plist failed: %s', str(ex))
def test_collect_path_events(self):
"""
Test path event collect before store.
"""
clang50_trunk_plist = os.path.join(self.__plist_test_files,
'clang-5.0-trunk.plist')
files, reports = plist_parser.parse_plist(clang50_trunk_plist, None,
False)
self.assertEqual(len(reports), 3)
# Generate dummy file_ids which should come from the database.
file_ids = {}
for i, file_name in enumerate(files, 1):
file_ids[file_name] = i
msg = "This test is prepared to handle 3 reports."
self.assertEqual(len(reports), 3, msg)
report1_path = [
ttypes.BugPathPos(startLine=19,
filePath=None,
endCol=7,
startCol=5,
endLine=19,
fileId=1),
ttypes.BugPathPos(startLine=20,
filePath=None,
endCol=7,
startCol=5,
endLine=20,
fileId=1),
ttypes.BugPathPos(startLine=21,
filePath=None,
endCol=13,
startCol=5,
endLine=21,
fileId=1),
ttypes.BugPathPos(startLine=7,
filePath=None,
endCol=7,
startCol=5,
endLine=7,
fileId=1),
ttypes.BugPathPos(startLine=8,
filePath=None,
endCol=6,
startCol=5,
endLine=8,
fileId=1),
ttypes.BugPathPos(startLine=8,
filePath=None,
endCol=25,
startCol=22,
endLine=8,
fileId=1),
ttypes.BugPathPos(startLine=8,
filePath=None,
endCol=20,
startCol=10,
endLine=8,
fileId=1),
ttypes.BugPathPos(startLine=7,
filePath=None,
endCol=14,
startCol=14,
endLine=7,
fileId=2)
]
report1_events = [
ttypes.BugPathEvent(startLine=20,
filePath=None,
endCol=12,
startCol=5,
msg="'base' initialized to 0",
endLine=20,
fileId=1),
ttypes.BugPathEvent(startLine=21,
filePath=None,
endCol=18,
startCol=15,
msg="Passing the value 0 via "
"1st parameter 'base'",
endLine=21,
fileId=1),
ttypes.BugPathEvent(startLine=21,
filePath=None,
endCol=19,
startCol=5,
msg="Calling 'test_func'",
endLine=21,
fileId=1),
ttypes.BugPathEvent(startLine=6,
filePath=None,
endCol=1,
startCol=1,
msg="Entered call from 'main'",
endLine=6,
fileId=1),
ttypes.BugPathEvent(startLine=8,
filePath=None,
endCol=25,
startCol=22,
msg="Passing the value 0 via "
"1st parameter 'num'",
endLine=8,
fileId=1),
ttypes.BugPathEvent(startLine=8,
filePath=None,
endCol=26,
startCol=10,
msg="Calling 'generate_id'",
endLine=8,
fileId=1),
ttypes.BugPathEvent(startLine=6,
filePath=None,
endCol=1,
startCol=1,
msg="Entered call from 'test_func'",
endLine=6,
fileId=2),
ttypes.BugPathEvent(startLine=7,
filePath=None,
endCol=17,
startCol=12,
msg='Division by zero',
endLine=7,
fileId=2)
]
path1, events1, _ = store_handler.collect_paths_events(
reports[0], file_ids, files)
self.assertEqual(path1, report1_path)
self.assertEqual(events1, report1_events)
report2_path = []
report2_events = [
ttypes.BugPathEvent(startLine=8,
filePath=None,
endCol=26,
startCol=10,
msg="Value stored to 'id' is "
"never read",
endLine=8,
fileId=1)
]
path2, events2, _ = store_handler.collect_paths_events(
reports[1], file_ids, files)
self.assertEqual(path2, report2_path)
self.assertEqual(events2, report2_events)
report3_path = [
ttypes.BugPathPos(startLine=14,
filePath=None,
endCol=6,
startCol=3,
endLine=14,
fileId=1),
ttypes.BugPathPos(startLine=15,
filePath=None,
endCol=3,
startCol=3,
endLine=15,
fileId=1),
ttypes.BugPathPos(startLine=16,
filePath=None,
endCol=1,
startCol=1,
endLine=16,
fileId=1)
]
report3_events = [
ttypes.BugPathEvent(startLine=14,
filePath=None,
endCol=29,
startCol=3,
msg="Address of stack memory associated"
" with local variable 'str'"
" is still referred to by the global "
"variable 'p' upon returning to the "
"caller. This will be a dangling "
"reference",
endLine=14,
fileId=1)
]
path, events, _ = store_handler.collect_paths_events(
reports[2], file_ids, files)
self.assertEqual(path, report3_path)
self.assertEqual(events, report3_events)
def test_empty_file(self):
"""Plist file is empty."""
empty_plist = os.path.join(self.__plist_test_files, 'empty_file')
files, reports = plist_parser.parse_plist(empty_plist, None, False)
self.assertEquals(files, [])
self.assertEquals(reports, [])
def test_collect_path_events(self):
"""
Test path event collect before store.
"""
clang50_trunk_plist = os.path.join(
self.__plist_test_files, 'clang-5.0-trunk.plist')
files, reports = plist_parser.parse_plist(clang50_trunk_plist, None,
False)
self.assertEqual(len(reports), 3)
# Generate dummy file_ids which should come from the database.
file_ids = {}
for i, file_name in enumerate(files, 1):
file_ids[file_name] = i
msg = "This test is prepared to handle 3 reports."
self.assertEqual(len(reports), 3, msg)
report1_path = [
ttypes.BugPathPos(startLine=19, filePath=None, endCol=7,
startCol=5, endLine=19, fileId=1),
ttypes.BugPathPos(startLine=20, filePath=None, endCol=7,
startCol=5, endLine=20, fileId=1),
ttypes.BugPathPos(startLine=21, filePath=None, endCol=13,
startCol=5, endLine=21, fileId=1),
ttypes.BugPathPos(startLine=7, filePath=None, endCol=7,
startCol=5, endLine=7, fileId=1),
ttypes.BugPathPos(startLine=8, filePath=None, endCol=6,
startCol=5, endLine=8, fileId=1),
ttypes.BugPathPos(startLine=8, filePath=None, endCol=25,
startCol=22, endLine=8, fileId=1),
ttypes.BugPathPos(startLine=8, filePath=None, endCol=20,
startCol=10, endLine=8, fileId=1),
ttypes.BugPathPos(startLine=7, filePath=None, endCol=14,
startCol=14, endLine=7, fileId=2)
]
report1_events = [
ttypes.BugPathEvent(startLine=20, filePath=None, endCol=12,
startCol=5, msg="'base' initialized to 0",
endLine=20, fileId=1),
ttypes.BugPathEvent(startLine=21, filePath=None, endCol=18,
startCol=15,
msg="Passing the value 0 via "
"1st parameter 'base'",
endLine=21, fileId=1),
ttypes.BugPathEvent(startLine=21, filePath=None, endCol=19,
startCol=5, msg="Calling 'test_func'",
endLine=21, fileId=1),
ttypes.BugPathEvent(startLine=6, filePath=None, endCol=1,
startCol=1, msg="Entered call from 'main'",
endLine=6, fileId=1),
ttypes.BugPathEvent(startLine=8, filePath=None, endCol=25,
startCol=22,
msg="Passing the value 0 via "
"1st parameter 'num'", endLine=8, fileId=1),
ttypes.BugPathEvent(startLine=8, filePath=None, endCol=26,
startCol=10, msg="Calling 'generate_id'",
endLine=8, fileId=1),
ttypes.BugPathEvent(startLine=6, filePath=None, endCol=1,
startCol=1,
msg="Entered call from 'test_func'",
endLine=6, fileId=2),
ttypes.BugPathEvent(startLine=7, filePath=None, endCol=17,
startCol=12, msg='Division by zero',
endLine=7, fileId=2)
]
path1, events1, _ = store_handler.collect_paths_events(reports[0],
file_ids,
files)
self.assertEqual(path1, report1_path)
self.assertEqual(events1, report1_events)
report2_path = []
report2_events = [
ttypes.BugPathEvent(startLine=8, filePath=None, endCol=26,
startCol=10,
msg="Value stored to 'id' is ""never read",
endLine=8, fileId=1)
]
path2, events2, _ = store_handler.collect_paths_events(reports[1],
file_ids,
files)
self.assertEqual(path2, report2_path)
self.assertEqual(events2, report2_events)
report3_path = [
ttypes.BugPathPos(startLine=14, filePath=None, endCol=6,
startCol=3, endLine=14, fileId=1),
ttypes.BugPathPos(startLine=15, filePath=None, endCol=3,
startCol=3, endLine=15, fileId=1),
ttypes.BugPathPos(startLine=16, filePath=None, endCol=1,
startCol=1, endLine=16, fileId=1)
]
report3_events = [
ttypes.BugPathEvent(startLine=14, filePath=None, endCol=29,
startCol=3,
msg="Address of stack memory associated"
" with local variable 'str'"
" is still referred to by the global "
"variable 'p' upon returning to the "
"caller. This will be a dangling "
"reference",
endLine=14, fileId=1)
]
path, events, _ = store_handler.collect_paths_events(reports[2],
file_ids,
files)
self.assertEqual(path, report3_path)
self.assertEqual(events, report3_events)