def test(self): instance = os.path.join(self.aux_root, "three_paths.c") aa_file = os.path.join(self.aux_root, "aa_three_paths_inner_if_both_blocks.spc") specs_dir = self.temp_folder with patch.object(self.logger, "info") as mock_info: g = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=specs_dir, cex_count=10, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer(), ) # Updating covered lines, to force the generator to cover # the only other possible path. g.lines_covered.update([12, 13, 14, 18]) cex_generated = [next(g.generate_executions())] # Updating covered lines, to reflect the execution just produced. g.lines_covered.update([12, 13, 14, 15]) cex_generated += list(g.generate_executions()) expected_calls = [ call("Generated 1 executions."), call("Generated 0 executions."), ] self.assertEqual(mock_info.mock_calls, expected_calls) self.assertTrue(os.path.exists(self.temp_folder)) self.assertEqual(len(os.listdir(self.temp_folder)), 2) self.assertEqual(len(cex_generated), 1)
def test(self): instance = os.path.join(self.aux_root, "three_paths.c") aa_file = os.path.join( self.aux_root, "aa_three_paths_else_return_not_covered.spc" ) with patch.object(self.logger, "info") as mock_info: c = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=self.temp_folder, cex_count=10, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer(), ) lines_covered, lines_to_cover = c.collect_coverage() expected_calls = [ call("Generated 1 executions."), call("Coverage after collecting 1 executions:"), call("Lines covered: 3"), call("Total lines to cover: 10"), call(""), call("Generated 0 executions."), call("Total lines covered: 3"), call("Total lines to cover: 10"), ] self.assertEqual(mock_info.mock_calls, expected_calls) self.assertEqual(lines_covered, {12, 13, 22}) self.assertEqual(lines_to_cover, {12, 13, 14, 15, 16, 18, 19, 22, 23, 24})
def test(self): instance = os.path.join(self.aux_root, "three_paths.c") aa_file = os.path.join(self.aux_root, "aa_three_paths_inner_if_both_blocks.spc") specs_dir = os.path.join(self.aux_root, "cex_three_paths", "inner_both_blocks") with patch.object(self.logger, "info") as mock_info: c = generate_coverage.CollectFromExistingExecutions( instance=instance, cex_dir=specs_dir, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer(), ) lines_covered, lines_to_cover = c.collect_coverage() expected_calls = [ call("Coverage after collecting 1 executions:"), call("Lines covered: 4"), call("Total lines to cover: 10"), call(""), call("Coverage after collecting 2 executions:"), call("Lines covered: 5"), call("Total lines to cover: 10"), call(""), call("Total lines covered: 5"), call("Total lines to cover: 10"), ] self.assertEqual(mock_info.mock_calls, expected_calls) self.assertEqual(lines_covered, {12, 13, 14, 15, 18}) self.assertEqual(lines_to_cover, {12, 13, 14, 15, 16, 18, 19, 22, 23, 24})
def test(self): instance = os.path.join(self.aux_root, 'multiline_switch_and_call.c') aa_file = os.path.join(self.aux_root, os.pardir, 'true_aa.spc') start_time = time.time() with patch.object(self.logger, 'info') as mock_info: c = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=self.temp_folder, cex_count=10, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=start_time, timer=generate_coverage.Timer()) lines_covered, lines_to_cover = \ c.collect_coverage() # Compound expression in cond (lines 3-4) is considered as fully # evaluated even though line 4 cannot be reached. self.assertEqual( lines_covered, set([3, 4, 7, 8, 9, 10, 11, 12, 18, 19, 20, 21, 22, 27, 28, 29])) self.assertEqual( lines_to_cover, set([ 3, 4, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34 ]))
def test(self): """ This is known behavior that is worth to document. We will only get a single execution for each return statement due to how states are merged. """ instance = os.path.join(self.aux_root, "one_per_return.c") cex_count = 10 aa_file = os.path.join(self.aux_root, "dummy_aa.spc") with patch.object(self.logger, "info") as mock_info: g = generate_coverage.GenerateFirstThenCollect( instance=instance, output_dir=self.temp_folder, cex_count=cex_count, spec=self.default_spec, heap_size=None, timelimit=self.default_timelimit, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer(), ) cex_generated = len(list(g.generate_executions())) mock_info.assert_called_once_with("Generated 1 executions.") self.assertTrue(os.path.exists(self.temp_folder)) self.assertEqual(len(os.listdir(self.temp_folder)), 1) self.assertEqual(cex_generated, 1)
def test(self): instance = os.path.join(self.aux_root, "three_paths.c") aa_file = os.path.join(self.aux_root, "aa_three_paths_inner_if_both_blocks.spc") specs_dir = self.temp_folder with patch.object(self.logger, "info") as mock_info: g = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=specs_dir, cex_count=10, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer(), ) # Updating covered lines such that it is impossible to # cover more lines. g.lines_covered.update([12, 13, 14, 15, 18]) cex_generated = list(g.generate_executions()) expected_calls = [call("Generated 0 executions.")] self.assertEqual(mock_info.mock_calls, expected_calls) self.assertTrue(os.path.exists(self.temp_folder)) self.assertEqual(len(os.listdir(self.temp_folder)), 0) self.assertEqual(len(cex_generated), 0)
def test(self): instance = os.path.join(self.aux_root, "three_paths.c") aa_file = os.path.join(self.aux_root, "aa_three_paths_else_return_not_covered.spc") specs_dir = os.path.join(self.aux_root, "cex_three_paths", "outer_else_block") with patch.object(self.logger, "info") as mock_info: c = generate_coverage.CollectFromExistingExecutions( instance=instance, cex_dir=specs_dir, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer(), ) lines_covered, lines_to_cover = c.collect_coverage() expected_calls = [ call("Coverage after collecting 1 executions:"), call("Lines covered: 3"), call("Total lines to cover: 10"), call(""), call("Total lines covered: 3"), call("Total lines to cover: 10"), ] self.assertEqual(mock_info.mock_calls, expected_calls) self.assertEqual(lines_covered, set([3, 4, 13])) self.assertEqual(lines_to_cover, set([3, 4, 5, 6, 7, 9, 10, 13, 14, 15]))
def test(self): instance = os.path.join(self.aux_root, 'three_paths.c') aa_file = os.path.join(self.aux_root, 'aa_three_paths_else_return_not_covered.spc') with patch.object(self.logger, 'info') as mock_info: c = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=self.temp_folder, cex_count=10, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer()) lines_covered, lines_to_cover = \ c.collect_coverage() expected_calls = [ call('Generated 1 executions.'), call('Coverage after collecting 1 executions:'), call('Lines covered: 3'), call('Total lines to cover: 10'), call(''), call('Generated 0 executions.'), call('Total lines covered: 3'), call('Total lines to cover: 10') ] self.assertEqual(mock_info.mock_calls, expected_calls) self.assertEqual(lines_covered, set([3, 4, 13])) self.assertEqual(lines_to_cover, set([3, 4, 5, 6, 7, 9, 10, 13, 14, 15]))
def test(self): instance = os.path.join(self.aux_root, 'three_paths.c') aa_file = os.path.join(self.aux_root, 'aa_three_paths_inner_if_both_blocks.spc') specs_dir = os.path.join(self.aux_root, 'cex_three_paths', 'inner_both_blocks') with patch.object(self.logger, 'info') as mock_info: c = generate_coverage.CollectFromExistingExecutions( instance=instance, cex_dir=specs_dir, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer()) lines_covered, lines_to_cover = \ c.collect_coverage() expected_calls = [ call('Coverage after collecting 1 executions:'), call('Lines covered: 4'), call('Total lines to cover: 10'), call(''), call('Coverage after collecting 2 executions:'), call('Lines covered: 5'), call('Total lines to cover: 10'), call(''), call('Total lines covered: 5'), call('Total lines to cover: 10') ] self.assertEqual(mock_info.mock_calls, expected_calls) self.assertEqual(lines_covered, set([3, 4, 5, 6, 9])) self.assertEqual(lines_to_cover, set([3, 4, 5, 6, 7, 9, 10, 13, 14, 15]))
def test(self): instance = os.path.join(self.aux_root, "switch.c") aa_file = os.path.join(self.aux_root, os.pardir, "true_aa.spc") start_time = time.time() with patch.object(self.logger, "info"): c = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=self.temp_folder, cex_count=10, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=start_time, timer=generate_coverage.Timer(), ) lines_covered, lines_to_cover = c.collect_coverage() self.assertEqual(lines_covered, {12, 13, 14, 17, 18, 23, 24, 25}) self.assertEqual( lines_to_cover, { 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 }, )
def test(self): instance = os.path.join(self.aux_root, "multiline_switch_and_call.c") aa_file = os.path.join(self.aux_root, os.pardir, "true_aa.spc") start_time = time.time() with patch.object(self.logger, "info"): c = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=self.temp_folder, cex_count=10, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=start_time, timer=generate_coverage.Timer(), ) lines_covered, lines_to_cover = c.collect_coverage() # Compound expression in cond (lines 3-4) is considered as fully # evaluated even though line 4 cannot be reached. self.assertEqual( lines_covered, {12, 13, 16, 17, 18, 19, 20, 21, 27, 28, 29, 30, 31, 36, 37, 38}, ) self.assertEqual(lines_to_cover, {12, 13} | (set(range(16, 44)) - {26}))
def test(self): instance = os.path.join(self.aux_root, 'switch.c') aa_file = os.path.join(self.aux_root, os.pardir, 'true_aa.spc') start_time = time.time() with patch.object(self.logger, 'info') as mock_info: c = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=self.temp_folder, cex_count=10, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=start_time, timer=generate_coverage.Timer()) lines_covered, lines_to_cover = \ c.collect_coverage() self.assertEqual(lines_covered, set([3, 4, 5, 8, 9, 14, 15, 16])) self.assertEqual( lines_to_cover, set([ 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 ]))
def test(self): instance = os.path.join(self.aux_root, "three_paths.c") aa_file = os.path.join(self.aux_root, "single_true_state.spc") specs_dir = self.temp_folder with patch.object(self.logger, "info") as mock_info: g = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=specs_dir, cex_count=10, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer(), ) cex_generated = [next(g.generate_executions())] # Updating covered lines, to force the generator to cover # other lines. g.lines_covered.update([12, 13, 22, 23, 24]) cex_generated.append(next(g.generate_executions())) g.lines_covered.update([12, 13, 14, 15, 16]) cex_generated.append(next(g.generate_executions())) g.lines_covered.update([12, 13, 14, 18, 19]) expected_calls = [ call("Generated 1 executions."), call("Generated 1 executions."), call("Generated 1 executions."), ] self.assertEqual(mock_info.mock_calls, expected_calls) self.assertEqual(len(list(g.generate_executions())), 0) self.assertTrue(os.path.exists(self.temp_folder)) self.assertEqual(len(os.listdir(self.temp_folder)), 6) self.assertEqual(len(cex_generated), 3)
def test(self): instance = os.path.join(self.aux_root, "contains_error.c") cex_count = 2 # Will only produce one, checking the output though. aa_file = os.path.join(self.aux_root, "dummy_aa.spc") with patch.object(self.logger, "error") as mock_logger, patch.object( self.logger, "info" ) as mock_info: try: g = generate_coverage.GenerateFirstThenCollect( instance=instance, output_dir=self.temp_folder, cex_count=cex_count, spec=self.default_spec, heap_size=None, timelimit=self.default_timelimit, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer(), ) list(g.generate_executions()) self.fail("Should have raised FoundBugError.") except generate_coverage.FoundBugError: pass mock_logger.assert_called_once_with( "Found an assertion violation. " "Inspect counterexamples before collecting a " "coverage measure." ) mock_info.assert_called_once_with("Generated 2 executions.") self.assertTrue(os.path.exists(self.temp_folder)) self.assertEqual(len(os.listdir(self.temp_folder)), 2)
def test(self): instance = os.path.join(self.aux_root, "if.c") aa_file = os.path.join(self.aux_root, os.pardir, "true_aa.spc") start_time = time.time() with patch.object(self.logger, "info") as mock_info: c = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=self.temp_folder, cex_count=10, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=start_time, timer=generate_coverage.Timer(), ) lines_covered, lines_to_cover = c.collect_coverage() self.assertEqual(lines_covered, set([3, 4])) self.assertEqual(lines_to_cover, set([3, 4, 5, 6, 7]))
def test(self): instance = os.path.join(self.aux_root, 'two_loop_iterations.c') cex_count = 2 # Will only produce one, checking the output though. aa_file = os.path.join(self.aux_root, 'dummy_aa.spc') with patch.object(self.logger, 'info') as mock_logger: g = generate_coverage.GenerateFirstThenCollect( instance=instance, output_dir=self.temp_folder, cex_count=cex_count, spec=self.default_spec, heap_size=None, timelimit=self.default_timelimit, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer()) cex_generated = len(list(g.generate_executions())) mock_logger.assert_called_once_with('Generated 1 executions.') self.assertTrue(os.path.exists(self.temp_folder)) self.assertEqual(len(os.listdir(self.temp_folder)), 1) self.assertEqual(cex_generated, 1)
def test(self): instance = os.path.join(self.aux_root, "three_paths.c") cex_count = 10 # There are only 3 paths though, checking the output. aa_file = os.path.join(self.aux_root, "dummy_aa.spc") with patch.object(self.logger, "info") as mock_info: g = generate_coverage.GenerateFirstThenCollect( instance=instance, output_dir=self.temp_folder, cex_count=cex_count, spec=self.default_spec, heap_size=None, timelimit=self.default_timelimit, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer(), ) cex_generated = len(list(g.generate_executions())) mock_info.assert_called_once_with("Generated 3 executions.") self.assertTrue(os.path.exists(self.temp_folder)) self.assertEqual(len(os.listdir(self.temp_folder)), 3) self.assertEqual(cex_generated, 3)
def test(self): instance = os.path.join(self.aux_root, "three_paths.c") aa_file = os.path.join(self.aux_root, "single_true_state.spc") specs_dir = self.temp_folder with patch.object(self.logger, "info") as mock_info: g = generate_coverage.FixPointOnCoveredLines( instance=instance, output_dir=specs_dir, cex_count=1, spec=self.default_spec, heap_size=None, timelimit=None, logger=self.logger, aa_file=aa_file, start_time=self.start_time, timer=generate_coverage.Timer(), ) cex_generated = list(g.generate_executions()) expected_calls = [call("Generated 1 executions.")] self.assertEqual(mock_info.mock_calls, expected_calls) self.assertTrue(os.path.exists(self.temp_folder)) self.assertEqual(len(os.listdir(self.temp_folder)), 2) self.assertEqual(len(cex_generated), 1)