def test_parse(self): """ Test if there are skip messages in the output of the parse command if the source files did change between the analysis and the parse. """ test_proj_path = self._testproject_data['project_path'] test_proj_files = os.listdir(test_proj_path) print(test_proj_files) null_deref_file = os.path.join(test_proj_path, 'null_dereference.cpp') codechecker.log_and_analyze(self._codechecker_cfg, test_proj_path) ret, out, _ = codechecker.parse(self._codechecker_cfg) self.assertEqual(ret, 0) # Need to wait a little before updating the last modification time. # If we do not wait, not enough time will be past # between the analysis and the parse in the test. time.sleep(2) touch(null_deref_file) ret, out, _ = codechecker.parse(self._codechecker_cfg) self.assertEqual(ret, 0) msg = 'did change since the last analysis.' self.assertTrue(msg in out, '"' + msg + '" was not found in the parse output')
def test_check_without_metadata(self): """ This test checks whether the storage works without a metadata.json. """ runs = self._cc_client.getRunData(None, None, 0, None) if runs: run_id = max(map(lambda run: run.runId, runs)) # Remove the run. self._cc_client.removeRun(run_id, None) self._create_source_file(0) codechecker.log_and_analyze(self._codechecker_cfg, self._test_dir) try: # Test storage without metadata.json. os.remove( os.path.join(self._codechecker_cfg['reportdir'], 'metadata.json')) except OSError: # metadata.json already removed. pass codechecker.store(self._codechecker_cfg, 'hello') runs = self._cc_client.getRunData(None, None, 0, None) run_id = max(map(lambda run: run.runId, runs)) reports = self._cc_client.getRunResults([run_id], 100, 0, [], None, None, False) self.assertEqual(len(reports), 2)
def test_store_multiple_dir_no_off(self): """ Store multiple report directory and check that no reports are marked as OFF. """ cfg = dict(self._codechecker_cfg) cfg['checkers'] = ['-d', 'core.DivideZero'] self._create_source_file(1) codechecker.log_and_analyze(cfg, self._test_dir) # Remove metadata.json. try: os.remove(os.path.join(cfg['reportdir'], 'metadata.json')) except OSError: pass # Analyze the same project to a different report directory and disable # modernize checkers. cfg['checkers'] = ['-d', 'deadcode.DeadStores'] cfg['reportdir'] = self._codechecker_cfg['reportdir'] + "2" orig_test_dir = self._test_dir self._test_dir = self._test_dir + "2" shutil.copytree(orig_test_dir, self._test_dir) self._create_source_file(3) codechecker.log_and_analyze(cfg, self._test_dir) # Set back test dir. self._test_dir = orig_test_dir # Store two report directory. cfg['reportdir'] = '{0} {1}'.format( cfg['reportdir'], self._codechecker_cfg['reportdir']) codechecker.store(cfg, 'hello') # Check that no reports are marked as OFF. reports = self._cc_client.getRunResults(None, 100, 0, [], None, None, False) offed_reports = [r for r in reports if r.detectionStatus == DetectionStatus.OFF] self.assertEqual(len(offed_reports), 0)
def setup_package(): """Setup the environment for the tests.""" global TEST_WORKSPACE TEST_WORKSPACE = env.get_workspace('suppress') os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE test_project = 'suppress' test_config = {} project_info = project.get_info(test_project) test_proj_path = os.path.join(TEST_WORKSPACE, "test_proj") shutil.copytree(project.path(test_project), test_proj_path) project_info['project_path'] = test_proj_path test_config['test_project'] = project_info # Generate a suppress file for the tests. suppress_file = os.path.join(TEST_WORKSPACE, 'suppress_file') if os.path.isfile(suppress_file): os.remove(suppress_file) _generate_suppress_file(suppress_file) test_env = env.test_env(TEST_WORKSPACE) codechecker_cfg = { 'suppress_file': None, 'skip_list_file': None, 'check_env': test_env, 'workspace': TEST_WORKSPACE, 'checkers': [] } ret = project.clean(test_project, test_env) if ret: sys.exit(ret) output_dir = codechecker_cfg['reportdir'] \ if 'reportdir' in codechecker_cfg \ else os.path.join(codechecker_cfg['workspace'], 'reports') codechecker_cfg['reportdir'] = output_dir ret = codechecker.log_and_analyze(codechecker_cfg, project.path(test_project)) if ret: sys.exit(1) print("Analyzing the test project was successful.") test_config['codechecker_cfg'] = codechecker_cfg env.export_test_cfg(TEST_WORKSPACE, test_config)
def test_suppress_reports(self): """ Check diff command when analysing the same source file which contains source code comments. """ cfg = dict(self._codechecker_cfg) cfg['analyzers'] = ['clang-tidy'] makefile = f"all:\n\t$(CXX) -c main.cpp -Wno-all -Wno-extra " \ f"-o /dev/null\n" with open(os.path.join(self._test_dir, 'Makefile'), 'w', encoding="utf-8", errors="ignore") as f: f.write(makefile) project_info = { "name": "suppress", "clean_cmd": "", "build_cmd": "make" } with open(os.path.join(self._test_dir, 'project_info.json'), 'w', encoding="utf-8", errors="ignore") as f: json.dump(project_info, f) # 1st phase. content = """ int main() { sizeof(41); sizeof(42); sizeof(43); }""" with open(os.path.join(self._test_dir, "main.cpp"), 'w', encoding="utf-8", errors="ignore") as f: f.write(content) report_dir_base = os.path.join(self._test_dir, "reports1") cfg['reportdir'] = report_dir_base codechecker.log_and_analyze(cfg, self._test_dir) # 2nd phase. content = """ int main() { // codechecker_intentional [all] This bug is suppressed in this change. sizeof(41); sizeof(42); // codechecker_confirmed [all] This bug is a real bug sizeof(44); sizeof(45); }""" with open(os.path.join(self._test_dir, "main.cpp"), 'w', encoding="utf-8", errors="ignore") as f: f.write(content) report_dir_new = os.path.join(self._test_dir, "reports2") cfg['reportdir'] = report_dir_new codechecker.log_and_analyze(cfg, self._test_dir) # Run the diff command and check the results. res, _, _ = get_diff_results([report_dir_base], [report_dir_new], '--new', 'json') print(res) self.assertEqual(len(res), 2) res, _, _ = get_diff_results([report_dir_base], [report_dir_new], '--unresolved', 'json') self.assertEqual(len(res), 1) res, _, _ = get_diff_results([report_dir_base], [report_dir_new], '--resolved', 'json') self.assertEqual(len(res), 2)
def setup_package(): """Setup the environment for testing diff_local_remote.""" global TEST_WORKSPACE TEST_WORKSPACE = env.get_workspace('diff_local_remote') # Set the TEST_WORKSPACE used by the tests. os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE # Get the clang version which is used for testing. # Important because different clang releases might # find different errors. clang_version = env.clang_to_test() test_config = {} test_project = 'cpp' project_info = project.get_info(test_project) # Copy the test project to the workspace. The tests should # work only on this test project. test_proj_path_local = os.path.join(TEST_WORKSPACE, "test_proj_local") shutil.copytree(project.path(test_project), test_proj_path_local) # Copy the test project to the workspace. The tests should # work only on this test project. test_proj_path_remote = os.path.join(TEST_WORKSPACE, "test_proj_remote") shutil.copytree(project.path(test_project), test_proj_path_remote) project_info['project_path_local'] = test_proj_path_local project_info['project_path_remote'] = test_proj_path_remote test_config['test_project'] = project_info # Suppress file should be set here if needed by the tests. suppress_file = None # Skip list file should be set here if needed by the tests. skip_list_file = None # Get an environment which should be used by the tests. test_env = env.test_env(TEST_WORKSPACE) # Create a basic CodeChecker config for the tests, this should # be imported by the tests and they should only depend on these # configuration options. codechecker_cfg = { 'suppress_file': suppress_file, 'skip_list_file': skip_list_file, 'check_env': test_env, 'workspace': TEST_WORKSPACE, 'checkers': [] } # Start or connect to the running CodeChecker server and get connection # details. print("This test uses a CodeChecker server... connecting...") server_access = codechecker.start_or_get_server() server_access['viewer_product'] = 'diff_local_remote' codechecker.add_test_package_product(server_access, TEST_WORKSPACE) # Extend the checker configuration with the server access. codechecker_cfg.update(server_access) # Analyze local, these reports will not be stored to the server. altered_file = os.path.join(test_proj_path_local, "call_and_message.cpp") project.insert_suppression(altered_file) codechecker_cfg['reportdir'] = os.path.join(test_proj_path_local, 'reports') codechecker_cfg['checkers'] = [ '-e', 'core.CallAndMessage', '-d', 'core.NullDereference' ] ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_local) if ret: sys.exit(1) print('Analyzing local was successful.') # Remote analysis, results will be stored to the remote server. altered_file = os.path.join(test_proj_path_local, "call_and_message.cpp") project.insert_suppression(altered_file) codechecker_cfg['reportdir'] = os.path.join(test_proj_path_remote, 'reports') codechecker_cfg['checkers'] = [ '-d', 'core.CallAndMessage', '-e', 'core.NullDereference' ] ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_remote) if ret: sys.exit(1) print('Analyzing new was successful.') # Store results to the remote server. test_project_name_remote = project_info['name'] + '_' + uuid.uuid4().hex ret = codechecker.store(codechecker_cfg, test_project_name_remote) if ret: sys.exit(1) print('Analyzing remote was successful.') # Save the run names in the configuration. codechecker_cfg['run_names'] = [test_project_name_remote] test_config['codechecker_cfg'] = codechecker_cfg # Export the test configuration to the workspace. env.export_test_cfg(TEST_WORKSPACE, test_config)
def setup_package(): """Setup the environment for testing diff_local.""" global TEST_WORKSPACE TEST_WORKSPACE = env.get_workspace('diff_local') # Set the TEST_WORKSPACE used by the tests. os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE test_config = {} test_project = 'cpp' project_info = project.get_info(test_project) # Copy the test project to the workspace. The tests should # work only on this test project. test_proj_path_base = os.path.join(TEST_WORKSPACE, "test_proj_base") shutil.copytree(project.path(test_project), test_proj_path_base) # Copy the test project to the workspace. The tests should # work only on this test project. test_proj_path_new = os.path.join(TEST_WORKSPACE, "test_proj_new") shutil.copytree(project.path(test_project), test_proj_path_new) project_info['project_path_base'] = test_proj_path_base project_info['project_path_new'] = test_proj_path_new test_config['test_project'] = project_info # Suppress file should be set here if needed by the tests. suppress_file = None # Skip list file should be set here if needed by the tests. skip_list_file = None # Get an environment which should be used by the tests. test_env = env.test_env(TEST_WORKSPACE) # Create a basic CodeChecker config for the tests, this should # be imported by the tests and they should only depend on these # configuration options. codechecker_cfg = { 'suppress_file': suppress_file, 'skip_list_file': skip_list_file, 'check_env': test_env, 'workspace': TEST_WORKSPACE, 'checkers': [] } # Base analysis codechecker_cfg['reportdir'] = os.path.join(test_proj_path_base, 'reports') codechecker_cfg['checkers'] = [ '-e', 'core.CallAndMessage', '-d', 'core.NullDereference' ] ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_base) if ret: sys.exit(1) # New analysis codechecker_cfg['reportdir'] = os.path.join(test_proj_path_new, 'reports') codechecker_cfg['checkers'] = [ '-d', 'core.CallAndMessage', '-e', 'core.NullDereference' ] ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_new) if ret: sys.exit(1) codechecker_cfg['reportdir_base'] = os.path.join(test_proj_path_base, 'reports') codechecker_cfg['reportdir_new'] = os.path.join(test_proj_path_new, 'reports') test_config['codechecker_cfg'] = codechecker_cfg # Export the test configuration to the workspace. env.export_test_cfg(TEST_WORKSPACE, test_config)
def setup_package(): """Setup the environment for the tests.""" global TEST_WORKSPACE TEST_WORKSPACE = env.get_workspace('skip') os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE test_project = 'cpp' test_config = {} project_info = project.get_info(test_project) test_config['test_project'] = project_info suppress_file = None # Generate skip list file for the tests. skip_list_file = os.path.join(TEST_WORKSPACE, 'skip_file') if os.path.isfile(skip_list_file): os.remove(skip_list_file) _generate_skip_list_file(skip_list_file) test_env = env.test_env(TEST_WORKSPACE) codechecker_cfg = { 'suppress_file': suppress_file, 'skip_file': skip_list_file, 'check_env': test_env, 'workspace': TEST_WORKSPACE, 'checkers': [] } ret = project.clean(test_project, test_env) if ret: sys.exit(ret) # Start or connect to the running CodeChecker server and get connection # details. print("This test uses a CodeChecker server... connecting...") server_access = codechecker.start_or_get_server() server_access['viewer_product'] = 'skip' codechecker.add_test_package_product(server_access, TEST_WORKSPACE) # Extend the checker configuration with the server access. codechecker_cfg.update(server_access) test_project_name = project_info['name'] + '_' + uuid.uuid4().hex skip_file = codechecker_cfg.pop('skip_file') output_dir = codechecker_cfg['reportdir'] \ if 'reportdir' in codechecker_cfg \ else os.path.join(codechecker_cfg['workspace'], 'reports') codechecker_cfg['reportdir'] = output_dir # Analyze without skip. ret = codechecker.log_and_analyze(codechecker_cfg, project.path(test_project)) if ret: print("Analyzing the test project without a skip file failed.") sys.exit(1) codechecker_cfg['skip_file'] = skip_file # Analyze with skip. ret = codechecker.log_and_analyze(codechecker_cfg, project.path(test_project)) if ret: print("Analyzing the test project with a skip file failed.") sys.exit(1) ret = codechecker.store(codechecker_cfg, test_project_name) if ret: print("Storing the results failed.") sys.exit(1) codechecker_cfg['run_names'] = [test_project_name] test_config['codechecker_cfg'] = codechecker_cfg env.export_test_cfg(TEST_WORKSPACE, test_config)
def setup_package(): """Setup the environment for testing diff_remote.""" global TEST_WORKSPACE TEST_WORKSPACE = env.get_workspace('diff_remote') # Set the TEST_WORKSPACE used by the tests. os.environ['TEST_WORKSPACE'] = TEST_WORKSPACE test_config = {} test_project = 'cpp' project_info = project.get_info(test_project) # Copy the test project to the workspace. The tests should # work only on this test project. test_proj_path_base = os.path.join(TEST_WORKSPACE, "test_proj_base") shutil.copytree(project.path(test_project), test_proj_path_base) # Copy the test project to the workspace. The tests should # work only on this test project. test_proj_path_new = os.path.join(TEST_WORKSPACE, "test_proj_new") shutil.copytree(project.path(test_project), test_proj_path_new) # Copy the test project to the workspace. The tests should # work only on this test project. test_proj_path_update = os.path.join(TEST_WORKSPACE, "test_proj_update") shutil.copytree(project.path(test_project), test_proj_path_update) project_info['project_path_base'] = test_proj_path_base project_info['project_path_new'] = test_proj_path_new project_info['project_path_update'] = test_proj_path_update # Suppress file should be set here if needed by the tests. suppress_file = None # Skip list file should be set here if needed by the tests. skip_list_file = None # Get an environment which should be used by the tests. test_env = env.test_env(TEST_WORKSPACE) # Create a basic CodeChecker config for the tests, this should # be imported by the tests and they should only depend on these # configuration options. codechecker_cfg = { 'suppress_file': suppress_file, 'skip_list_file': skip_list_file, 'check_env': test_env, 'workspace': TEST_WORKSPACE, 'checkers': [] } # Start or connect to the running CodeChecker server and get connection # details. print("This test uses a CodeChecker server... connecting...") server_access = codechecker.start_or_get_server() server_access['viewer_product'] = 'diff_remote' codechecker.add_test_package_product(server_access, TEST_WORKSPACE) # Extend the checker configuration with the server access. codechecker_cfg.update(server_access) # Base analysis altered_file = os.path.join(test_proj_path_base, "call_and_message.cpp") project.insert_suppression(altered_file) codechecker_cfg['reportdir'] = os.path.join(test_proj_path_base, 'reports') codechecker_cfg['checkers'] = ['-e', 'core.CallAndMessage', '-d', 'core.NullDereference'] ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_base) if ret: sys.exit(1) print('Analyzing base was successful.') # Store base results. codechecker_cfg['reportdir'] = os.path.join(test_proj_path_base, 'reports') test_project_name_base = project_info['name'] + '_' + uuid.uuid4().hex ret = codechecker.store(codechecker_cfg, test_project_name_base) if ret: sys.exit(1) # Store with a literal ':' in the name. ret = codechecker.store(codechecker_cfg, test_project_name_base + ":base") if ret: sys.exit(1) # New analysis altered_file = os.path.join(test_proj_path_new, "call_and_message.cpp") project.insert_suppression(altered_file) codechecker_cfg['reportdir'] = os.path.join(test_proj_path_new, 'reports') codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', '-e', 'core.NullDereference'] ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_new) if ret: sys.exit(1) print('Analyzing new was successful.') # Store new results. codechecker_cfg['reportdir'] = os.path.join(test_proj_path_new, 'reports') test_project_name_new = project_info['name'] + '_' + uuid.uuid4().hex ret = codechecker.store(codechecker_cfg, test_project_name_new) if ret: sys.exit(1) # Store with a literal ':' in the name. ret = codechecker.store(codechecker_cfg, test_project_name_new + ":new") if ret: sys.exit(1) # Analyze multiple times to store results with multiple tags. codechecker_cfg['reportdir'] = os.path.join(test_proj_path_update, 'reports') test_project_name_update = project_info['name'] + '_' + uuid.uuid4().hex codechecker_cfg['tag'] = 't1' codechecker_cfg['checkers'] = ['-d', 'core.CallAndMessage', '-e', 'core.StackAddressEscape' ] codechecker_cfg['reportdir'] = os.path.join(test_proj_path_update, 'reports') ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_update) if ret: sys.exit(1) # Store update with t1 tag. ret = codechecker.store(codechecker_cfg, test_project_name_update) if ret: sys.exit(1) codechecker_cfg['tag'] = 't2' codechecker_cfg['checkers'] = ['-e', 'core.CallAndMessage', '-d', 'core.StackAddressEscape' ] ret = codechecker.analyze(codechecker_cfg, test_proj_path_update) if ret: sys.exit(1) # Store update with t2 tag. ret = codechecker.store(codechecker_cfg, test_project_name_update) if ret: sys.exit(1) codechecker_cfg['tag'] = 't3' ret = codechecker.log_and_analyze(codechecker_cfg, test_proj_path_update) if ret: sys.exit(1) # Store update with t3 tag. ret = codechecker.store(codechecker_cfg, test_project_name_update) if ret: sys.exit(1) # Order of the test run names matter at comparison! codechecker_cfg['run_names'] = [test_project_name_base, test_project_name_new, test_project_name_update] test_config['test_project'] = project_info test_config['codechecker_cfg'] = codechecker_cfg # Export the test configuration to the workspace. env.export_test_cfg(TEST_WORKSPACE, test_config) # Remove report directories which are not used anymore. shutil.rmtree(test_proj_path_base, ignore_errors=True) shutil.rmtree(test_proj_path_new, ignore_errors=True)