def test(self):
instance = os.path.join(self.aux_root, "three_paths.c")
aa_file = os.path.join(self.aux_root, "aa_three_paths_inner_if_both_blocks.spc")
specs_dir = self.temp_folder
with patch.object(self.logger, "info") as mock_info:
g = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=specs_dir,
cex_count=10,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer(),
)
# Updating covered lines, to force the generator to cover
# the only other possible path.
g.lines_covered.update([12, 13, 14, 18])
cex_generated = [next(g.generate_executions())]
# Updating covered lines, to reflect the execution just produced.
g.lines_covered.update([12, 13, 14, 15])
cex_generated += list(g.generate_executions())
expected_calls = [
call("Generated 1 executions."),
call("Generated 0 executions."),
]
self.assertEqual(mock_info.mock_calls, expected_calls)
self.assertTrue(os.path.exists(self.temp_folder))
self.assertEqual(len(os.listdir(self.temp_folder)), 2)
self.assertEqual(len(cex_generated), 1)
def test(self):
instance = os.path.join(self.aux_root, "three_paths.c")
aa_file = os.path.join(
self.aux_root, "aa_three_paths_else_return_not_covered.spc"
)
with patch.object(self.logger, "info") as mock_info:
c = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=self.temp_folder,
cex_count=10,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer(),
)
lines_covered, lines_to_cover = c.collect_coverage()
expected_calls = [
call("Generated 1 executions."),
call("Coverage after collecting 1 executions:"),
call("Lines covered: 3"),
call("Total lines to cover: 10"),
call(""),
call("Generated 0 executions."),
call("Total lines covered: 3"),
call("Total lines to cover: 10"),
]
self.assertEqual(mock_info.mock_calls, expected_calls)
self.assertEqual(lines_covered, {12, 13, 22})
self.assertEqual(lines_to_cover, {12, 13, 14, 15, 16, 18, 19, 22, 23, 24})
def test(self):
instance = os.path.join(self.aux_root, "three_paths.c")
aa_file = os.path.join(self.aux_root, "aa_three_paths_inner_if_both_blocks.spc")
specs_dir = os.path.join(self.aux_root, "cex_three_paths", "inner_both_blocks")
with patch.object(self.logger, "info") as mock_info:
c = generate_coverage.CollectFromExistingExecutions(
instance=instance,
cex_dir=specs_dir,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer(),
)
lines_covered, lines_to_cover = c.collect_coverage()
expected_calls = [
call("Coverage after collecting 1 executions:"),
call("Lines covered: 4"),
call("Total lines to cover: 10"),
call(""),
call("Coverage after collecting 2 executions:"),
call("Lines covered: 5"),
call("Total lines to cover: 10"),
call(""),
call("Total lines covered: 5"),
call("Total lines to cover: 10"),
]
self.assertEqual(mock_info.mock_calls, expected_calls)
self.assertEqual(lines_covered, {12, 13, 14, 15, 18})
self.assertEqual(lines_to_cover, {12, 13, 14, 15, 16, 18, 19, 22, 23, 24})
def test(self):
instance = os.path.join(self.aux_root, 'multiline_switch_and_call.c')
aa_file = os.path.join(self.aux_root, os.pardir, 'true_aa.spc')
start_time = time.time()
with patch.object(self.logger, 'info') as mock_info:
c = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=self.temp_folder,
cex_count=10,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=start_time,
timer=generate_coverage.Timer())
lines_covered, lines_to_cover = \
c.collect_coverage()
# Compound expression in cond (lines 3-4) is considered as fully
# evaluated even though line 4 cannot be reached.
self.assertEqual(
lines_covered,
set([3, 4, 7, 8, 9, 10, 11, 12, 18, 19, 20, 21, 22, 27, 28, 29]))
self.assertEqual(
lines_to_cover,
set([
3, 4, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22,
23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34
]))
def test(self):
"""
This is known behavior that is worth to document. We will only get
a single execution for each return statement due to how states
are merged.
"""
instance = os.path.join(self.aux_root, "one_per_return.c")
cex_count = 10
aa_file = os.path.join(self.aux_root, "dummy_aa.spc")
with patch.object(self.logger, "info") as mock_info:
g = generate_coverage.GenerateFirstThenCollect(
instance=instance,
output_dir=self.temp_folder,
cex_count=cex_count,
spec=self.default_spec,
heap_size=None,
timelimit=self.default_timelimit,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer(),
)
cex_generated = len(list(g.generate_executions()))
mock_info.assert_called_once_with("Generated 1 executions.")
self.assertTrue(os.path.exists(self.temp_folder))
self.assertEqual(len(os.listdir(self.temp_folder)), 1)
self.assertEqual(cex_generated, 1)
def test(self):
instance = os.path.join(self.aux_root, "three_paths.c")
aa_file = os.path.join(self.aux_root, "aa_three_paths_inner_if_both_blocks.spc")
specs_dir = self.temp_folder
with patch.object(self.logger, "info") as mock_info:
g = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=specs_dir,
cex_count=10,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer(),
)
# Updating covered lines such that it is impossible to
# cover more lines.
g.lines_covered.update([12, 13, 14, 15, 18])
cex_generated = list(g.generate_executions())
expected_calls = [call("Generated 0 executions.")]
self.assertEqual(mock_info.mock_calls, expected_calls)
self.assertTrue(os.path.exists(self.temp_folder))
self.assertEqual(len(os.listdir(self.temp_folder)), 0)
self.assertEqual(len(cex_generated), 0)
def test(self):
instance = os.path.join(self.aux_root, "three_paths.c")
aa_file = os.path.join(self.aux_root,
"aa_three_paths_else_return_not_covered.spc")
specs_dir = os.path.join(self.aux_root, "cex_three_paths",
"outer_else_block")
with patch.object(self.logger, "info") as mock_info:
c = generate_coverage.CollectFromExistingExecutions(
instance=instance,
cex_dir=specs_dir,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer(),
)
lines_covered, lines_to_cover = c.collect_coverage()
expected_calls = [
call("Coverage after collecting 1 executions:"),
call("Lines covered: 3"),
call("Total lines to cover: 10"),
call(""),
call("Total lines covered: 3"),
call("Total lines to cover: 10"),
]
self.assertEqual(mock_info.mock_calls, expected_calls)
self.assertEqual(lines_covered, set([3, 4, 13]))
self.assertEqual(lines_to_cover,
set([3, 4, 5, 6, 7, 9, 10, 13, 14, 15]))
def test(self):
instance = os.path.join(self.aux_root, 'three_paths.c')
aa_file = os.path.join(self.aux_root,
'aa_three_paths_else_return_not_covered.spc')
with patch.object(self.logger, 'info') as mock_info:
c = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=self.temp_folder,
cex_count=10,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer())
lines_covered, lines_to_cover = \
c.collect_coverage()
expected_calls = [
call('Generated 1 executions.'),
call('Coverage after collecting 1 executions:'),
call('Lines covered: 3'),
call('Total lines to cover: 10'),
call(''),
call('Generated 0 executions.'),
call('Total lines covered: 3'),
call('Total lines to cover: 10')
]
self.assertEqual(mock_info.mock_calls, expected_calls)
self.assertEqual(lines_covered, set([3, 4, 13]))
self.assertEqual(lines_to_cover,
set([3, 4, 5, 6, 7, 9, 10, 13, 14, 15]))
def test(self):
instance = os.path.join(self.aux_root, 'three_paths.c')
aa_file = os.path.join(self.aux_root,
'aa_three_paths_inner_if_both_blocks.spc')
specs_dir = os.path.join(self.aux_root, 'cex_three_paths',
'inner_both_blocks')
with patch.object(self.logger, 'info') as mock_info:
c = generate_coverage.CollectFromExistingExecutions(
instance=instance,
cex_dir=specs_dir,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer())
lines_covered, lines_to_cover = \
c.collect_coverage()
expected_calls = [
call('Coverage after collecting 1 executions:'),
call('Lines covered: 4'),
call('Total lines to cover: 10'),
call(''),
call('Coverage after collecting 2 executions:'),
call('Lines covered: 5'),
call('Total lines to cover: 10'),
call(''),
call('Total lines covered: 5'),
call('Total lines to cover: 10')
]
self.assertEqual(mock_info.mock_calls, expected_calls)
self.assertEqual(lines_covered, set([3, 4, 5, 6, 9]))
self.assertEqual(lines_to_cover,
set([3, 4, 5, 6, 7, 9, 10, 13, 14, 15]))
def test(self):
instance = os.path.join(self.aux_root, "switch.c")
aa_file = os.path.join(self.aux_root, os.pardir, "true_aa.spc")
start_time = time.time()
with patch.object(self.logger, "info"):
c = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=self.temp_folder,
cex_count=10,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=start_time,
timer=generate_coverage.Timer(),
)
lines_covered, lines_to_cover = c.collect_coverage()
self.assertEqual(lines_covered, {12, 13, 14, 17, 18, 23, 24, 25})
self.assertEqual(
lines_to_cover,
{
12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
29, 30
},
)
def test(self):
instance = os.path.join(self.aux_root, "multiline_switch_and_call.c")
aa_file = os.path.join(self.aux_root, os.pardir, "true_aa.spc")
start_time = time.time()
with patch.object(self.logger, "info"):
c = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=self.temp_folder,
cex_count=10,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=start_time,
timer=generate_coverage.Timer(),
)
lines_covered, lines_to_cover = c.collect_coverage()
# Compound expression in cond (lines 3-4) is considered as fully
# evaluated even though line 4 cannot be reached.
self.assertEqual(
lines_covered,
{12, 13, 16, 17, 18, 19, 20, 21, 27, 28, 29, 30, 31, 36, 37, 38},
)
self.assertEqual(lines_to_cover,
{12, 13} | (set(range(16, 44)) - {26}))
def test(self):
instance = os.path.join(self.aux_root, 'switch.c')
aa_file = os.path.join(self.aux_root, os.pardir, 'true_aa.spc')
start_time = time.time()
with patch.object(self.logger, 'info') as mock_info:
c = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=self.temp_folder,
cex_count=10,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=start_time,
timer=generate_coverage.Timer())
lines_covered, lines_to_cover = \
c.collect_coverage()
self.assertEqual(lines_covered, set([3, 4, 5, 8, 9, 14, 15, 16]))
self.assertEqual(
lines_to_cover,
set([
3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21
]))
def test(self):
instance = os.path.join(self.aux_root, "three_paths.c")
aa_file = os.path.join(self.aux_root, "single_true_state.spc")
specs_dir = self.temp_folder
with patch.object(self.logger, "info") as mock_info:
g = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=specs_dir,
cex_count=10,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer(),
)
cex_generated = [next(g.generate_executions())]
# Updating covered lines, to force the generator to cover
# other lines.
g.lines_covered.update([12, 13, 22, 23, 24])
cex_generated.append(next(g.generate_executions()))
g.lines_covered.update([12, 13, 14, 15, 16])
cex_generated.append(next(g.generate_executions()))
g.lines_covered.update([12, 13, 14, 18, 19])
expected_calls = [
call("Generated 1 executions."),
call("Generated 1 executions."),
call("Generated 1 executions."),
]
self.assertEqual(mock_info.mock_calls, expected_calls)
self.assertEqual(len(list(g.generate_executions())), 0)
self.assertTrue(os.path.exists(self.temp_folder))
self.assertEqual(len(os.listdir(self.temp_folder)), 6)
self.assertEqual(len(cex_generated), 3)
def test(self):
instance = os.path.join(self.aux_root, "contains_error.c")
cex_count = 2 # Will only produce one, checking the output though.
aa_file = os.path.join(self.aux_root, "dummy_aa.spc")
with patch.object(self.logger, "error") as mock_logger, patch.object(
self.logger, "info"
) as mock_info:
try:
g = generate_coverage.GenerateFirstThenCollect(
instance=instance,
output_dir=self.temp_folder,
cex_count=cex_count,
spec=self.default_spec,
heap_size=None,
timelimit=self.default_timelimit,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer(),
)
list(g.generate_executions())
self.fail("Should have raised FoundBugError.")
except generate_coverage.FoundBugError:
pass
mock_logger.assert_called_once_with(
"Found an assertion violation. "
"Inspect counterexamples before collecting a "
"coverage measure."
)
mock_info.assert_called_once_with("Generated 2 executions.")
self.assertTrue(os.path.exists(self.temp_folder))
self.assertEqual(len(os.listdir(self.temp_folder)), 2)
def test(self):
instance = os.path.join(self.aux_root, "if.c")
aa_file = os.path.join(self.aux_root, os.pardir, "true_aa.spc")
start_time = time.time()
with patch.object(self.logger, "info") as mock_info:
c = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=self.temp_folder,
cex_count=10,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=start_time,
timer=generate_coverage.Timer(),
)
lines_covered, lines_to_cover = c.collect_coverage()
self.assertEqual(lines_covered, set([3, 4]))
self.assertEqual(lines_to_cover, set([3, 4, 5, 6, 7]))
def test(self):
instance = os.path.join(self.aux_root, 'two_loop_iterations.c')
cex_count = 2 # Will only produce one, checking the output though.
aa_file = os.path.join(self.aux_root, 'dummy_aa.spc')
with patch.object(self.logger, 'info') as mock_logger:
g = generate_coverage.GenerateFirstThenCollect(
instance=instance,
output_dir=self.temp_folder,
cex_count=cex_count,
spec=self.default_spec,
heap_size=None,
timelimit=self.default_timelimit,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer())
cex_generated = len(list(g.generate_executions()))
mock_logger.assert_called_once_with('Generated 1 executions.')
self.assertTrue(os.path.exists(self.temp_folder))
self.assertEqual(len(os.listdir(self.temp_folder)), 1)
self.assertEqual(cex_generated, 1)
def test(self):
instance = os.path.join(self.aux_root, "three_paths.c")
cex_count = 10 # There are only 3 paths though, checking the output.
aa_file = os.path.join(self.aux_root, "dummy_aa.spc")
with patch.object(self.logger, "info") as mock_info:
g = generate_coverage.GenerateFirstThenCollect(
instance=instance,
output_dir=self.temp_folder,
cex_count=cex_count,
spec=self.default_spec,
heap_size=None,
timelimit=self.default_timelimit,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer(),
)
cex_generated = len(list(g.generate_executions()))
mock_info.assert_called_once_with("Generated 3 executions.")
self.assertTrue(os.path.exists(self.temp_folder))
self.assertEqual(len(os.listdir(self.temp_folder)), 3)
self.assertEqual(cex_generated, 3)
def test(self):
instance = os.path.join(self.aux_root, "three_paths.c")
aa_file = os.path.join(self.aux_root, "single_true_state.spc")
specs_dir = self.temp_folder
with patch.object(self.logger, "info") as mock_info:
g = generate_coverage.FixPointOnCoveredLines(
instance=instance,
output_dir=specs_dir,
cex_count=1,
spec=self.default_spec,
heap_size=None,
timelimit=None,
logger=self.logger,
aa_file=aa_file,
start_time=self.start_time,
timer=generate_coverage.Timer(),
)
cex_generated = list(g.generate_executions())
expected_calls = [call("Generated 1 executions.")]
self.assertEqual(mock_info.mock_calls, expected_calls)
self.assertTrue(os.path.exists(self.temp_folder))
self.assertEqual(len(os.listdir(self.temp_folder)), 2)
self.assertEqual(len(cex_generated), 1)