def test_filter_unique(self):
"""
Get all results by unique and non unique filter and check the results.
"""
sort_types = None
simple_filter = ReportFilter()
unique_filter = ReportFilter(isUnique=True)
# Get unique results.
run_results = self._cc_client.getRunResults(None, 500, 0, sort_types,
unique_filter, None, False)
unique_result_count = self._cc_client.getRunResultCount(
None, unique_filter, None)
unique_bughash = set([res.bugHash for res in run_results])
# Get simple results.
run_results = self._cc_client.getRunResults(None, 500, 0, sort_types,
simple_filter, None, False)
simple_result_count = self._cc_client.getRunResultCount(
None, simple_filter, None)
simple_bughash = set([res.bugHash for res in run_results])
diff_hashes = list(simple_bughash.difference(unique_bughash))
self.assertEqual(0, len(diff_hashes))
self.assertGreaterEqual(simple_result_count, unique_result_count)
def test_filter_report_by_component(self):
"""
Test report filter by component.
"""
components = self._cc_client.getSourceComponents(None)
self.assertEqual(len(components), 0)
# Create a new source component.
ret = self._cc_client.addSourceComponent(self._component_name,
self._component_value,
self._component_description)
self.assertTrue(ret)
r_filter = ReportFilter(componentNames=[self._component_name])
run_results = self._cc_client.getRunResults(None, 500, 0, None,
r_filter, None)
self.assertIsNotNone(run_results)
divide_zero_reports = [
r for r in run_results if r.checkedFile.endswith('divide_zero.cpp')
]
self.assertNotEqual(len(divide_zero_reports), 0)
divide_zero_reports = [
r for r in run_results if r.checkedFile.endswith('new_delete.cpp')
]
self.assertEqual(len(divide_zero_reports), 0)
def test_bug_path_length(self):
runid = self._runid
sortMode1 = SortMode(SortType.BUG_PATH_LENGTH, Order.ASC)
sortMode2 = SortMode(SortType.BUG_PATH_LENGTH, Order.DESC)
simple_filter = ReportFilter()
unique_filter = ReportFilter(isUnique=True)
run_results = self._cc_client.getRunResults([runid], 100, 0,
[sortMode1], simple_filter,
None, False)
self.__check_bug_path_order(run_results, Order.ASC)
run_results = self._cc_client.getRunResults([runid], 100, 0,
[sortMode2], unique_filter,
None, False)
self.__check_bug_path_order(run_results, Order.DESC)
def test_get_diff_res_count_unresolved_filter(self):
base_run_id = self._base_runid
new_run_id = self._new_runid
filter_severity_levels = [{
"MEDIUM": 1
}, {
"LOW": 6
}, {
"HIGH": 15
}, {
"STYLE": 0
}, {
"UNSPECIFIED": 3
}, {
"CRITICAL": 0
}]
cmp_data = CompareData(runIds=[new_run_id],
diffType=DiffType.UNRESOLVED)
for level in filter_severity_levels:
for severity_level, test_result_count in level.items():
sev = get_severity_level(severity_level)
sev_filter = ReportFilter(severity=[sev])
diff_result_count = self._cc_client.getRunResultCount(
[base_run_id], sev_filter, cmp_data)
self.assertEqual(test_result_count, diff_result_count)
def test_run1_run2_file_filters(self):
"""
Get all the core checker counts for run1 and run2.
"""
null_stack_filter = ReportFilter(
filepath=["*null_dereference.cpp", "*stack_address_escape.cpp"])
file_counts = self._cc_client.getFileCounts(self._runids,
null_stack_filter,
None,
None,
0)
res = {get_filename(k): v for k, v in file_counts.items()}
null_r1 = {k: v for k, v in self.run1_files.items()
if "null_dereference" in k}
stack_r1 = {k: v for k, v in self.run1_files.items()
if "stack_address_escape" in k}
null_r2 = {k: v for k, v in self.run2_files.items()
if "null_dereference" in k}
stack_r2 = {k: v for k, v in self.run2_files.items()
if "stack_address_escape" in k}
test_res = dict(Counter(null_r1) + Counter(null_r2) +
Counter(stack_r1) + Counter(stack_r2))
self.assertGreaterEqual(len(res), len(test_res))
self.assertDictContainsSubset(test_res, res)
def test_get_diff_res_types_unresolved_filter(self):
"""
Test diff result types for unresolved results with filter.
"""
base_run_id = self._base_runid
new_run_id = self._new_runid
diff_res_types_filter = self._testproject_data[self._clang_to_test][
'diff_res_types_filter']
cmp_data = CompareData(runIds=[new_run_id],
diffType=DiffType.UNRESOLVED)
for level in diff_res_types_filter:
for checker_name, test_result_count in level.items():
checker_filter = ReportFilter(checkerName=[checker_name])
diff_res = \
self._cc_client.getCheckerCounts([base_run_id],
checker_filter,
cmp_data,
None,
0)
diff_dict = dict((res.name, res.count) for res in diff_res)
# There should be only one result for each checker name.
self.assertEqual(test_result_count, diff_dict[checker_name])
def test_bug_path_length_filter(self):
"""
Filter by bug path length.
"""
sort_types = None
filters = [(None, 1), (1, 2), (1, None)]
for filter_values in filters:
len_min = filter_values[0]
len_max = filter_values[1]
bug_path_len_filter = BugPathLengthRange(min=len_min, max=len_max)
f = ReportFilter(bugPathLength=bug_path_len_filter)
run_results = self._cc_client.getRunResults(
self._runids, None, 0, sort_types, f, None, False)
self.assertIsNotNone(run_results)
if len_max:
self.assertTrue(
all(r.bugPathLength <= len_max for r in run_results))
if len_min:
self.assertTrue(
all(r.bugPathLength >= len_min for r in run_results))
def test_diff_run_tags(self):
"""Test for diff runs by run tags."""
run_id = self._update_runid
def get_run_tag_id(tag_name):
run_history_filter = RunHistoryFilter(tagNames=[tag_name])
run_tags = self._cc_client.getRunHistory([run_id], None, None,
run_history_filter)
self.assertEqual(len(run_tags), 1)
return run_tags[0].id
base_tag_id = get_run_tag_id('t1')
new_tag_id = get_run_tag_id('t2')
tag_filter = ReportFilter(runTag=[base_tag_id])
cmp_data = CompareData(runIds=[run_id],
diffType=DiffType.NEW,
runTag=[new_tag_id])
diff_res = self._cc_client.getRunResults([run_id], 500, 0, [],
tag_filter, cmp_data, False)
# 5 new core.CallAndMessage issues.
self.assertEqual(len(diff_res), 5)
cmp_data.diffType = DiffType.RESOLVED
diff_res = self._cc_client.getRunResults([run_id], 500, 0, [],
tag_filter, cmp_data, False)
self.assertEqual(len(diff_res), 3)
cmp_data.diffType = DiffType.UNRESOLVED
diff_res = self._cc_client.getRunResults([run_id], 500, 0, [],
tag_filter, cmp_data, False)
self.assertEqual(len(diff_res), 26)
def test_number_of_unique_reports(self):
"""
Tests that resultCount field value in runData is equal with the
number of unfixed reports in the run.
"""
test_runs = self.__get_runs()
report_filter = ReportFilter()
report_filter.detectionStatus = [DetectionStatus.NEW,
DetectionStatus.UNRESOLVED,
DetectionStatus.REOPENED]
for run in test_runs:
run_count = self._cc_client.getRunResultCount([run.runId],
report_filter,
None)
self.assertEqual(run.resultCount, run_count)
def test_number_of_unique_reports(self):
"""
Tests that resultCount field value in runData is equal with the
number of unfixed reports in the run.
"""
test_runs = self.__get_runs()
report_filter = ReportFilter()
report_filter.detectionStatus = [DetectionStatus.NEW,
DetectionStatus.UNRESOLVED,
DetectionStatus.REOPENED]
for run in test_runs:
run_count = self._cc_client.getRunResultCount([run.runId],
report_filter,
None)
self.assertEqual(run.resultCount, run_count)
def __get_file_id_for_path(self, run_id, file_path):
# Returns file id for the given file path in the given run.
file_filter = ReportFilter(filepath=[file_path])
run_results = self._cc_client.getRunResults([run_id], 1, 0, None,
file_filter, None, False)
self.assertTrue(len(run_results))
return run_results[0].fileId
def test_report_details(self):
"""
Get run results and check that report details are correctly set.
"""
runid = self._runid
simple_filter = ReportFilter()
run_results = self._cc_client.getRunResults([runid], 100, 0, None,
simple_filter, None, True)
self.assertTrue(any(res.details for res in run_results))
def test_number_of_unique_reports(self):
"""
Tests that resultCount field value in runData is equal with the
number of unique reports in the run.
"""
test_runs = self.__get_runs()
unique_filter = ReportFilter(isUnique=True)
for run in test_runs:
run_count = self._cc_client.getRunResultCount([run.runId],
unique_filter, None)
self.assertEqual(run.resultCount, run_count)
def test_get_source_file_content(self):
"""
Test getting the source file content stored to the database.
Test unicode support the stored file can be decoded properly
compare results form the database to the original file.
"""
runid = self._runid
report_filter = ReportFilter(checkerName=['*'], filepath=['*.c*'])
run_result_count = self._cc_client.getRunResultCount([runid],
report_filter,
None)
self.assertTrue(run_result_count)
run_results = get_all_run_results(self._cc_client,
runid,
[],
report_filter)
self.assertIsNotNone(run_results)
for run_res in run_results:
self.assertTrue(re.match(r'.*\.c(pp)?$', run_res.checkedFile))
logging.debug('Getting the content of ' + run_res.checkedFile)
file_data = self._cc_client.getSourceFileData(run_res.fileId,
True,
None)
self.assertIsNotNone(file_data)
file_content1 = file_data.fileContent
self.assertIsNotNone(file_content1)
with codecs.open(run_res.checkedFile, 'r', 'UTF-8',
'replace') as source_file:
file_content2 = source_file.read()
self.assertEqual(file_content1, file_content2)
file_data_b64 = self._cc_client.getSourceFileData(
run_res.fileId, True, Encoding.BASE64)
self.assertIsNotNone(file_data_b64)
file_content1_b64 = base64.b64decode(file_data_b64.fileContent)
self.assertIsNotNone(file_content1_b64)
file_content2 = codecs.encode(file_content2, 'utf-8')
self.assertEqual(file_content1_b64, file_content2)
logging.debug('got ' + str(len(run_results)) + ' files')
self.assertEqual(run_result_count, len(run_results))
def test_filter_report_by_multiple_components(self):
"""
Test report filter by multiple components.
"""
test_component1 = self.components[3]
test_component2 = self.components[4]
components = self._cc_client.getSourceComponents(None)
self.assertEqual(len(components), 0)
self.__add_new_component(test_component1)
self.__add_new_component(test_component2)
components = self._cc_client.getSourceComponents(None)
self.assertEqual(len(components), 2)
r_filter = ReportFilter(
componentNames=[test_component1['name'], test_component2['name']])
run_results = self._cc_client.getRunResults(None, 500, 0, None,
r_filter, None, False)
self.assertIsNotNone(run_results)
# Check that reports which can be found in file what the source
# component including are in the filtered results.
divide_zero_reports = [
r for r in run_results if r.checkedFile.endswith('divide_zero.cpp')
]
self.assertNotEqual(len(divide_zero_reports), 0)
null_deref_reports = [
r for r in run_results
if r.checkedFile.endswith('null_dereference.cpp')
]
self.assertNotEqual(len(null_deref_reports), 0)
# Check that reports which can be found in file what the source
# component excluding are in the filtered results.
new_delete_reports = [
r for r in run_results if r.checkedFile.endswith('new_delete.cpp')
]
self.assertEqual(len(new_delete_reports), 0)
call_and_msg_reports = [
r for r in run_results
if r.checkedFile.endswith('call_and_message.cpp')
]
self.assertEqual(len(call_and_msg_reports), 0)
self.__remove_source_component(test_component1['name'])
self.__remove_source_component(test_component2['name'])
def test_filter_report_by_complex_component(self):
"""
Test report filter by complex component which includes and excludes
multiple file paths.
"""
test_component = self.components[2]
components = self._cc_client.getSourceComponents(None)
self.assertEqual(len(components), 0)
self.__add_new_component(test_component)
r_filter = ReportFilter(componentNames=[test_component['name']])
run_results = self._cc_client.getRunResults(None, 500, 0, None,
r_filter, None, False)
self.assertIsNotNone(run_results)
# Check that reports which can be found in file what the source
# component including are in the filtered results.
divide_zero_reports = [
r for r in run_results if r.checkedFile.endswith('divide_zero.cpp')
]
self.assertNotEqual(len(divide_zero_reports), 0)
null_deref_reports = [
r for r in run_results
if r.checkedFile.endswith('null_dereference.cpp')
]
self.assertNotEqual(len(null_deref_reports), 0)
# Check that reports which can be found in file what the source
# component excluding are in the filtered results.
new_delete_reports = [
r for r in run_results if r.checkedFile.endswith('new_delete.cpp')
]
self.assertEqual(len(new_delete_reports), 0)
call_and_msg_reports = [
r for r in run_results
if r.checkedFile.endswith('call_and_message.cpp')
]
self.assertEqual(len(call_and_msg_reports), 0)
# No reports should be listed which is not in an inclusion or if
# it is in an exclusion filter.
divide_zero_reports = [
r for r in run_results if r.checkedFile.endswith('path_begin.cpp')
]
self.assertEqual(len(divide_zero_reports), 0)
self.__remove_source_component(test_component['name'])
def test_filter_empty(self):
""" Filter value is empty list should return all results."""
runid = self._runids[0]
sort_types = None
f = ReportFilter()
run_result_count = self._cc_client.getRunResultCount([runid], f, None)
self.assertIsNotNone(run_result_count)
run_results = self._cc_client.getRunResults([runid], run_result_count,
0, sort_types, f, None,
False)
self.assertIsNotNone(run_results)
self.assertEqual(run_result_count, len(run_results))
def test_run1_detection_status_repoened(self):
"""
Get all the reopened checker counts for run1.
"""
runid = self._runids[0]
reopen_filter = ReportFilter(
detectionStatus=[DetectionStatus.REOPENED])
reopened_reports = self._cc_client.getCheckerCounts([runid],
reopen_filter,
None,
None,
0)
checkers_dict = dict((res.name, res.count) for res in reopened_reports)
self.assertDictEqual({}, checkers_dict)
def test_get_diff_checker_counts_core_resolved(self):
"""
Test diff result types for resolved core checker counts.
"""
base_run_id = self._base_runid
new_run_id = self._new_runid
report_filter = ReportFilter(checkerName=["*core*"])
cmp_data = CompareData(runIds=[new_run_id], diffType=DiffType.RESOLVED)
diff_res = self._cc_client.getCheckerCounts([base_run_id],
report_filter, cmp_data,
None, 0)
diff_dict = dict((res.name, res.count) for res in diff_res)
# Resolved core checkers.
test_res = {'core.CallAndMessage': 5}
self.assertDictEqual(diff_dict, test_res)
def test_get_diff_checker_counts_core_new(self):
"""
Test diff result types for new core checker counts.
"""
base_run_id = self._base_runid
new_run_id = self._new_runid
cmp_data = CompareData(runIds=[new_run_id], diffType=DiffType.NEW)
report_filter = ReportFilter(checkerName=["*core*"])
diff_res = self._cc_client.getCheckerCounts([base_run_id],
report_filter, cmp_data,
None, 0)
diff_dict = dict((res.name, res.count) for res in diff_res)
# core.CallAndMessage is the new checker.
test_res = {"core.NullDereference": 4}
self.assertDictEqual(diff_dict, test_res)
def test_remove_run_results(self):
"""
Test for removing run results and run.
"""
# Run the anaysis again with different setup.
test_project_path = self._testproject_data['project_path']
ret = project.clean(test_project_path)
if ret:
sys.exit(ret)
codechecker.check(self._codechecker_cfg,
'remove_run_results',
test_project_path)
run_filter = RunFilter(names=['remove_run_results'], exactMatch=True)
runs = self._cc_client.getRunData(run_filter)
self.assertEqual(len(runs), 1)
run_id = runs[0].runId
orig_results_count = \
self._cc_client.getRunResultCount([run_id], None, None)
self.assertNotEqual(orig_results_count, 0)
checker_filter = ReportFilter(checkerName=["core.CallAndMessage"])
res_count = self._cc_client.getRunResultCount([run_id],
checker_filter,
None)
self.assertNotEqual(res_count, 0)
self._cc_client.removeRunReports([run_id],
checker_filter,
None)
res_count = self._cc_client.getRunResultCount([run_id],
checker_filter,
None)
self.assertEqual(res_count, 0)
# Remove the run.
self._cc_client.removeRun(run_id)
# Check that we removed all results from the run.
res = self._cc_client.getRunResultCount([run_id], None, None)
self.assertEqual(res, 0)
def test_run1_detection_status_unresolved(self):
"""
Get all the unresolved checker counts for run1.
"""
runid = self._runids[0]
unresolved_filter = ReportFilter(
detectionStatus=[DetectionStatus.UNRESOLVED])
unresolved_reports = \
self._cc_client.getCheckerCounts([runid],
unresolved_filter,
None,
None,
0)
checkers_dict = dict((res.name, res.count)
for res in unresolved_reports)
self.assertDictEqual({}, checkers_dict)
def test_run1_core_checkers(self):
"""
Get all the core checker counts for run1.
"""
runid = self._runids[0]
core_filter = ReportFilter(checkerName=["core*"])
checker_counts = self._cc_client.getCheckerCounts([runid],
core_filter,
None,
None,
0)
checkers_dict = dict((res.name, res.count) for res in checker_counts)
core_checkers = {k: v for k, v in self.run1_checkers.items()
if "core." in k}
self.assertGreaterEqual(len(checker_counts), len(core_checkers))
self.assertDictContainsSubset(core_checkers, checkers_dict)
def test_get_diff_checker_counts_core_unresolved(self):
"""
Test diff result types for resolved unix checker counts.
"""
base_run_id = self._base_runid
new_run_id = self._new_runid
report_filter = ReportFilter(checkerName=["*core*"])
cmp_data = CompareData(runIds=[new_run_id],
diffType=DiffType.UNRESOLVED)
diff_res = self._cc_client.getCheckerCounts([base_run_id],
report_filter, cmp_data,
None, 0)
diff_dict = dict((res.name, res.count) for res in diff_res)
# Unresolved core checkers.
test_res = {'core.StackAddrEscapeBase': 3, 'core.DivideZero': 10}
self.assertDictContainsSubset(test_res, diff_dict)
def test_filter_report_by_excluding_all_results_component(self):
"""
Test report filter by component which excludes all reports.
"""
test_component = self.components[5]
components = self._cc_client.getSourceComponents(None)
self.assertEqual(len(components), 0)
self.__add_new_component(test_component)
r_filter = ReportFilter(componentNames=[test_component['name']])
run_results = self._cc_client.getRunResults(None, 500, 0, None,
r_filter, None, False)
self.assertIsNotNone(run_results)
# No reports for this component.
self.assertEqual(len(run_results), 0)
self.__remove_source_component(test_component['name'])
def test_uniqueing_compared_to_test_config(self):
"""
In the test config there are all of the reports without
uniqueing. This test checks if the uniqueing works for the
getRunResultCount api call.
"""
runid = self._runids[0]
bugs = self._testproject_data[self._clang_to_test]['bugs']
unique_filter = ReportFilter(isUnique=True)
run_result_count = self._cc_client.getRunResultCount([runid],
unique_filter,
None)
unique_bugs = set()
# Uniqueing is done based on bug hash.
for b in bugs:
unique_bugs.add((b['hash']))
self.assertEqual(len(unique_bugs), run_result_count)
def test_filter_checker_id(self):
""" Filter by checker id. """
runid = self._runids[0]
severity_test_data = self._testproject_data[
self._clang_to_test]['filter_checker_id']
for level in severity_test_data:
for checker_id_filter, test_result_count in level.items():
logging.debug('Checker id filter ' + checker_id_filter +
' test result count: ' + str(test_result_count))
sort_types = None
cid_f = ReportFilter(checkerName=[checker_id_filter])
run_results = self._cc_client.getRunResults([runid], 500, 0,
sort_types, cid_f,
None, False)
for r in run_results:
print(r)
self.assertIsNotNone(run_results)
self.assertEqual(test_result_count, len(run_results))
def test_filter_report_by_component(self):
"""
Test report filter by component.
"""
test_component = self.components[0]
components = self._cc_client.getSourceComponents(None)
self.assertEqual(len(components), 0)
self.__add_new_component(test_component)
r_filter = ReportFilter(componentNames=[test_component['name']])
run_results = self._cc_client.getRunResults(None,
500,
0,
None,
r_filter,
None)
self.assertIsNotNone(run_results)
# Check that reports which can be found in file what the source
# component including are in the filtered results.
divide_zero_reports = [r for r in run_results if
r.checkedFile.endswith('divide_zero.cpp')]
self.assertNotEqual(len(divide_zero_reports), 0)
# Check that reports which can be found in file what the source
# component excluding are in the filtered results.
new_delete_reports = [r for r in run_results if
r.checkedFile.endswith('new_delete.cpp')]
self.assertEqual(len(new_delete_reports), 0)
# Check that reports which can be found in file what the source
# component does not includes or excludes are in the filtered results.
divide_zero_reports = [r for r in run_results if
r.checkedFile.endswith('null_dereference.cpp')]
self.assertNotEqual(len(divide_zero_reports), 0)
self.__remove_source_component(test_component['name'])
def test_run1_detection_status_new(self):
"""
Get all the new checker counts for run1.
"""
runid = self._runids[0]
new_filter = ReportFilter(
detectionStatus=[DetectionStatus.NEW])
new_reports = self._cc_client.getCheckerCounts([runid],
new_filter,
None,
None,
0)
checkers_dict = dict((res.name, res.count) for res in new_reports)
new = {'core.CallAndMessage': 5,
'core.StackAddrEscapeBase': 3,
'cplusplus.NewDelete': 5,
'core.NullDereference': 4,
'core.DivideZero': 10,
'deadcode.DeadStores': 6,
'unix.Malloc': 1}
self.assertDictEqual(new, checkers_dict)
def test_filter_severity(self):
""" Filter by severity levels."""
runid = self._runids[0]
severity_test_data = self._testproject_data[
self._clang_to_test]['filter_severity_levels']
for level in severity_test_data:
for severity_level, test_result_count in level.items():
logging.debug('Severity level filter ' + severity_level +
' test result count: ' + str(test_result_count))
sort_types = None
sev = get_severity_level(severity_level)
sev_f = ReportFilter(severity=[sev])
run_result_count = self._cc_client.getRunResultCount([runid],
sev_f,
None)
run_results = self._cc_client.getRunResults([runid],
run_result_count,
0, sort_types,
sev_f, None, False)
self.assertIsNotNone(run_results)
self.assertEqual(test_result_count, len(run_results))
def test_run1_run2_core_checkers(self):
"""
Get all the core checker counts for run1 and run2.
"""
core_filter = ReportFilter(checkerName=["core*"])
checker_counts = self._cc_client.getCheckerCounts(self._runids,
core_filter,
None,
None,
0)
checkers_dict = dict((res.name, res.count) for res in checker_counts)
core_checkers_r1 = {k: v for k, v in self.run1_checkers.items()
if "core." in k}
core_checkers_r2 = {k: v for k, v in self.run2_checkers.items()
if "core." in k}
r1_core = Counter(core_checkers_r1)
r2_core = Counter(core_checkers_r2)
all_core = dict(r1_core + r2_core)
self.assertGreaterEqual(len(checker_counts), len(all_core))
self.assertDictContainsSubset(all_core, checkers_dict)