Exemplo n.º 1
0
 def test(self):
     instance = os.path.join(self.aux_root, "three_paths.c")
     aa_file = os.path.join(self.aux_root, "aa_three_paths_inner_if_both_blocks.spc")
     specs_dir = self.temp_folder
     with patch.object(self.logger, "info") as mock_info:
         g = generate_coverage.FixPointOnCoveredLines(
             instance=instance,
             output_dir=specs_dir,
             cex_count=10,
             spec=self.default_spec,
             heap_size=None,
             timelimit=None,
             logger=self.logger,
             aa_file=aa_file,
             start_time=self.start_time,
             timer=generate_coverage.Timer(),
         )
         # Updating covered lines, to force the generator to cover
         # the only other possible path.
         g.lines_covered.update([12, 13, 14, 18])
         cex_generated = [next(g.generate_executions())]
         # Updating covered lines, to reflect the execution just produced.
         g.lines_covered.update([12, 13, 14, 15])
         cex_generated += list(g.generate_executions())
         expected_calls = [
             call("Generated 1 executions."),
             call("Generated 0 executions."),
         ]
         self.assertEqual(mock_info.mock_calls, expected_calls)
     self.assertTrue(os.path.exists(self.temp_folder))
     self.assertEqual(len(os.listdir(self.temp_folder)), 2)
     self.assertEqual(len(cex_generated), 1)
Exemplo n.º 2
0
    def test(self):
        instance = os.path.join(self.aux_root, "three_paths.c")
        aa_file = os.path.join(
            self.aux_root, "aa_three_paths_else_return_not_covered.spc"
        )
        with patch.object(self.logger, "info") as mock_info:
            c = generate_coverage.FixPointOnCoveredLines(
                instance=instance,
                output_dir=self.temp_folder,
                cex_count=10,
                spec=self.default_spec,
                heap_size=None,
                timelimit=None,
                logger=self.logger,
                aa_file=aa_file,
                start_time=self.start_time,
                timer=generate_coverage.Timer(),
            )
            lines_covered, lines_to_cover = c.collect_coverage()
            expected_calls = [
                call("Generated 1 executions."),
                call("Coverage after collecting 1 executions:"),
                call("Lines covered: 3"),
                call("Total lines to cover: 10"),
                call(""),
                call("Generated 0 executions."),
                call("Total lines covered: 3"),
                call("Total lines to cover: 10"),
            ]
            self.assertEqual(mock_info.mock_calls, expected_calls)

        self.assertEqual(lines_covered, {12, 13, 22})
        self.assertEqual(lines_to_cover, {12, 13, 14, 15, 16, 18, 19, 22, 23, 24})
Exemplo n.º 3
0
    def test(self):
        instance = os.path.join(self.aux_root, "three_paths.c")
        aa_file = os.path.join(self.aux_root, "aa_three_paths_inner_if_both_blocks.spc")
        specs_dir = os.path.join(self.aux_root, "cex_three_paths", "inner_both_blocks")
        with patch.object(self.logger, "info") as mock_info:
            c = generate_coverage.CollectFromExistingExecutions(
                instance=instance,
                cex_dir=specs_dir,
                heap_size=None,
                timelimit=None,
                logger=self.logger,
                aa_file=aa_file,
                start_time=self.start_time,
                timer=generate_coverage.Timer(),
            )
            lines_covered, lines_to_cover = c.collect_coverage()
            expected_calls = [
                call("Coverage after collecting 1 executions:"),
                call("Lines covered: 4"),
                call("Total lines to cover: 10"),
                call(""),
                call("Coverage after collecting 2 executions:"),
                call("Lines covered: 5"),
                call("Total lines to cover: 10"),
                call(""),
                call("Total lines covered: 5"),
                call("Total lines to cover: 10"),
            ]
            self.assertEqual(mock_info.mock_calls, expected_calls)

        self.assertEqual(lines_covered, {12, 13, 14, 15, 18})
        self.assertEqual(lines_to_cover, {12, 13, 14, 15, 16, 18, 19, 22, 23, 24})
Exemplo n.º 4
0
 def test(self):
     instance = os.path.join(self.aux_root, 'multiline_switch_and_call.c')
     aa_file = os.path.join(self.aux_root, os.pardir, 'true_aa.spc')
     start_time = time.time()
     with patch.object(self.logger, 'info') as mock_info:
         c = generate_coverage.FixPointOnCoveredLines(
             instance=instance,
             output_dir=self.temp_folder,
             cex_count=10,
             spec=self.default_spec,
             heap_size=None,
             timelimit=None,
             logger=self.logger,
             aa_file=aa_file,
             start_time=start_time,
             timer=generate_coverage.Timer())
         lines_covered, lines_to_cover = \
             c.collect_coverage()
     # Compound expression in cond (lines 3-4) is considered as fully
     # evaluated even though line 4 cannot be reached.
     self.assertEqual(
         lines_covered,
         set([3, 4, 7, 8, 9, 10, 11, 12, 18, 19, 20, 21, 22, 27, 28, 29]))
     self.assertEqual(
         lines_to_cover,
         set([
             3, 4, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22,
             23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34
         ]))
Exemplo n.º 5
0
    def test(self):
        """
        This is known behavior that is worth to document. We will only get
        a single execution for each return statement due to how states
        are merged.
        """
        instance = os.path.join(self.aux_root, "one_per_return.c")
        cex_count = 10
        aa_file = os.path.join(self.aux_root, "dummy_aa.spc")
        with patch.object(self.logger, "info") as mock_info:
            g = generate_coverage.GenerateFirstThenCollect(
                instance=instance,
                output_dir=self.temp_folder,
                cex_count=cex_count,
                spec=self.default_spec,
                heap_size=None,
                timelimit=self.default_timelimit,
                logger=self.logger,
                aa_file=aa_file,
                start_time=self.start_time,
                timer=generate_coverage.Timer(),
            )
            cex_generated = len(list(g.generate_executions()))
            mock_info.assert_called_once_with("Generated 1 executions.")

        self.assertTrue(os.path.exists(self.temp_folder))
        self.assertEqual(len(os.listdir(self.temp_folder)), 1)
        self.assertEqual(cex_generated, 1)
Exemplo n.º 6
0
 def test(self):
     instance = os.path.join(self.aux_root, "three_paths.c")
     aa_file = os.path.join(self.aux_root, "aa_three_paths_inner_if_both_blocks.spc")
     specs_dir = self.temp_folder
     with patch.object(self.logger, "info") as mock_info:
         g = generate_coverage.FixPointOnCoveredLines(
             instance=instance,
             output_dir=specs_dir,
             cex_count=10,
             spec=self.default_spec,
             heap_size=None,
             timelimit=None,
             logger=self.logger,
             aa_file=aa_file,
             start_time=self.start_time,
             timer=generate_coverage.Timer(),
         )
         # Updating covered lines such that it is impossible to
         # cover more lines.
         g.lines_covered.update([12, 13, 14, 15, 18])
         cex_generated = list(g.generate_executions())
         expected_calls = [call("Generated 0 executions.")]
         self.assertEqual(mock_info.mock_calls, expected_calls)
     self.assertTrue(os.path.exists(self.temp_folder))
     self.assertEqual(len(os.listdir(self.temp_folder)), 0)
     self.assertEqual(len(cex_generated), 0)
Exemplo n.º 7
0
    def test(self):
        instance = os.path.join(self.aux_root, "three_paths.c")
        aa_file = os.path.join(self.aux_root,
                               "aa_three_paths_else_return_not_covered.spc")
        specs_dir = os.path.join(self.aux_root, "cex_three_paths",
                                 "outer_else_block")

        with patch.object(self.logger, "info") as mock_info:
            c = generate_coverage.CollectFromExistingExecutions(
                instance=instance,
                cex_dir=specs_dir,
                heap_size=None,
                timelimit=None,
                logger=self.logger,
                aa_file=aa_file,
                start_time=self.start_time,
                timer=generate_coverage.Timer(),
            )
            lines_covered, lines_to_cover = c.collect_coverage()
            expected_calls = [
                call("Coverage after collecting 1 executions:"),
                call("Lines covered: 3"),
                call("Total lines to cover: 10"),
                call(""),
                call("Total lines covered: 3"),
                call("Total lines to cover: 10"),
            ]
            self.assertEqual(mock_info.mock_calls, expected_calls)

        self.assertEqual(lines_covered, set([3, 4, 13]))
        self.assertEqual(lines_to_cover,
                         set([3, 4, 5, 6, 7, 9, 10, 13, 14, 15]))
    def test(self):
        instance = os.path.join(self.aux_root, 'three_paths.c')
        aa_file = os.path.join(self.aux_root,
                               'aa_three_paths_else_return_not_covered.spc')
        with patch.object(self.logger, 'info') as mock_info:
            c = generate_coverage.FixPointOnCoveredLines(
                instance=instance,
                output_dir=self.temp_folder,
                cex_count=10,
                spec=self.default_spec,
                heap_size=None,
                timelimit=None,
                logger=self.logger,
                aa_file=aa_file,
                start_time=self.start_time,
                timer=generate_coverage.Timer())
            lines_covered, lines_to_cover = \
                c.collect_coverage()
            expected_calls = [
                call('Generated 1 executions.'),
                call('Coverage after collecting 1 executions:'),
                call('Lines covered: 3'),
                call('Total lines to cover: 10'),
                call(''),
                call('Generated 0 executions.'),
                call('Total lines covered: 3'),
                call('Total lines to cover: 10')
            ]
            self.assertEqual(mock_info.mock_calls, expected_calls)

        self.assertEqual(lines_covered, set([3, 4, 13]))
        self.assertEqual(lines_to_cover,
                         set([3, 4, 5, 6, 7, 9, 10, 13, 14, 15]))
    def test(self):
        instance = os.path.join(self.aux_root, 'three_paths.c')
        aa_file = os.path.join(self.aux_root,
                               'aa_three_paths_inner_if_both_blocks.spc')
        specs_dir = os.path.join(self.aux_root, 'cex_three_paths',
                                 'inner_both_blocks')
        with patch.object(self.logger, 'info') as mock_info:
            c = generate_coverage.CollectFromExistingExecutions(
                instance=instance,
                cex_dir=specs_dir,
                heap_size=None,
                timelimit=None,
                logger=self.logger,
                aa_file=aa_file,
                start_time=self.start_time,
                timer=generate_coverage.Timer())
            lines_covered, lines_to_cover = \
                c.collect_coverage()
            expected_calls = [
                call('Coverage after collecting 1 executions:'),
                call('Lines covered: 4'),
                call('Total lines to cover: 10'),
                call(''),
                call('Coverage after collecting 2 executions:'),
                call('Lines covered: 5'),
                call('Total lines to cover: 10'),
                call(''),
                call('Total lines covered: 5'),
                call('Total lines to cover: 10')
            ]
            self.assertEqual(mock_info.mock_calls, expected_calls)

        self.assertEqual(lines_covered, set([3, 4, 5, 6, 9]))
        self.assertEqual(lines_to_cover,
                         set([3, 4, 5, 6, 7, 9, 10, 13, 14, 15]))
Exemplo n.º 10
0
    def test(self):
        instance = os.path.join(self.aux_root, "switch.c")
        aa_file = os.path.join(self.aux_root, os.pardir, "true_aa.spc")
        start_time = time.time()
        with patch.object(self.logger, "info"):
            c = generate_coverage.FixPointOnCoveredLines(
                instance=instance,
                output_dir=self.temp_folder,
                cex_count=10,
                spec=self.default_spec,
                heap_size=None,
                timelimit=None,
                logger=self.logger,
                aa_file=aa_file,
                start_time=start_time,
                timer=generate_coverage.Timer(),
            )
            lines_covered, lines_to_cover = c.collect_coverage()

        self.assertEqual(lines_covered, {12, 13, 14, 17, 18, 23, 24, 25})
        self.assertEqual(
            lines_to_cover,
            {
                12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
                29, 30
            },
        )
Exemplo n.º 11
0
 def test(self):
     instance = os.path.join(self.aux_root, "multiline_switch_and_call.c")
     aa_file = os.path.join(self.aux_root, os.pardir, "true_aa.spc")
     start_time = time.time()
     with patch.object(self.logger, "info"):
         c = generate_coverage.FixPointOnCoveredLines(
             instance=instance,
             output_dir=self.temp_folder,
             cex_count=10,
             spec=self.default_spec,
             heap_size=None,
             timelimit=None,
             logger=self.logger,
             aa_file=aa_file,
             start_time=start_time,
             timer=generate_coverage.Timer(),
         )
         lines_covered, lines_to_cover = c.collect_coverage()
     # Compound expression in cond (lines 3-4) is considered as fully
     # evaluated even though line 4 cannot be reached.
     self.assertEqual(
         lines_covered,
         {12, 13, 16, 17, 18, 19, 20, 21, 27, 28, 29, 30, 31, 36, 37, 38},
     )
     self.assertEqual(lines_to_cover,
                      {12, 13} | (set(range(16, 44)) - {26}))
Exemplo n.º 12
0
    def test(self):
        instance = os.path.join(self.aux_root, 'switch.c')
        aa_file = os.path.join(self.aux_root, os.pardir, 'true_aa.spc')
        start_time = time.time()
        with patch.object(self.logger, 'info') as mock_info:
            c = generate_coverage.FixPointOnCoveredLines(
                instance=instance,
                output_dir=self.temp_folder,
                cex_count=10,
                spec=self.default_spec,
                heap_size=None,
                timelimit=None,
                logger=self.logger,
                aa_file=aa_file,
                start_time=start_time,
                timer=generate_coverage.Timer())
            lines_covered, lines_to_cover = \
                c.collect_coverage()

        self.assertEqual(lines_covered, set([3, 4, 5, 8, 9, 14, 15, 16]))
        self.assertEqual(
            lines_to_cover,
            set([
                3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
                21
            ]))
Exemplo n.º 13
0
 def test(self):
     instance = os.path.join(self.aux_root, "three_paths.c")
     aa_file = os.path.join(self.aux_root, "single_true_state.spc")
     specs_dir = self.temp_folder
     with patch.object(self.logger, "info") as mock_info:
         g = generate_coverage.FixPointOnCoveredLines(
             instance=instance,
             output_dir=specs_dir,
             cex_count=10,
             spec=self.default_spec,
             heap_size=None,
             timelimit=None,
             logger=self.logger,
             aa_file=aa_file,
             start_time=self.start_time,
             timer=generate_coverage.Timer(),
         )
         cex_generated = [next(g.generate_executions())]
         # Updating covered lines, to force the generator to cover
         # other lines.
         g.lines_covered.update([12, 13, 22, 23, 24])
         cex_generated.append(next(g.generate_executions()))
         g.lines_covered.update([12, 13, 14, 15, 16])
         cex_generated.append(next(g.generate_executions()))
         g.lines_covered.update([12, 13, 14, 18, 19])
         expected_calls = [
             call("Generated 1 executions."),
             call("Generated 1 executions."),
             call("Generated 1 executions."),
         ]
         self.assertEqual(mock_info.mock_calls, expected_calls)
         self.assertEqual(len(list(g.generate_executions())), 0)
     self.assertTrue(os.path.exists(self.temp_folder))
     self.assertEqual(len(os.listdir(self.temp_folder)), 6)
     self.assertEqual(len(cex_generated), 3)
Exemplo n.º 14
0
    def test(self):
        instance = os.path.join(self.aux_root, "contains_error.c")
        cex_count = 2  # Will only produce one, checking the output though.
        aa_file = os.path.join(self.aux_root, "dummy_aa.spc")

        with patch.object(self.logger, "error") as mock_logger, patch.object(
            self.logger, "info"
        ) as mock_info:
            try:
                g = generate_coverage.GenerateFirstThenCollect(
                    instance=instance,
                    output_dir=self.temp_folder,
                    cex_count=cex_count,
                    spec=self.default_spec,
                    heap_size=None,
                    timelimit=self.default_timelimit,
                    logger=self.logger,
                    aa_file=aa_file,
                    start_time=self.start_time,
                    timer=generate_coverage.Timer(),
                )
                list(g.generate_executions())
                self.fail("Should have raised FoundBugError.")
            except generate_coverage.FoundBugError:
                pass
            mock_logger.assert_called_once_with(
                "Found an assertion violation. "
                "Inspect counterexamples before collecting a "
                "coverage measure."
            )
            mock_info.assert_called_once_with("Generated 2 executions.")

        self.assertTrue(os.path.exists(self.temp_folder))
        self.assertEqual(len(os.listdir(self.temp_folder)), 2)
Exemplo n.º 15
0
    def test(self):
        instance = os.path.join(self.aux_root, "if.c")
        aa_file = os.path.join(self.aux_root, os.pardir, "true_aa.spc")
        start_time = time.time()
        with patch.object(self.logger, "info") as mock_info:
            c = generate_coverage.FixPointOnCoveredLines(
                instance=instance,
                output_dir=self.temp_folder,
                cex_count=10,
                spec=self.default_spec,
                heap_size=None,
                timelimit=None,
                logger=self.logger,
                aa_file=aa_file,
                start_time=start_time,
                timer=generate_coverage.Timer(),
            )
            lines_covered, lines_to_cover = c.collect_coverage()

        self.assertEqual(lines_covered, set([3, 4]))
        self.assertEqual(lines_to_cover, set([3, 4, 5, 6, 7]))
    def test(self):
        instance = os.path.join(self.aux_root, 'two_loop_iterations.c')
        cex_count = 2  # Will only produce one, checking the output though.
        aa_file = os.path.join(self.aux_root, 'dummy_aa.spc')
        with patch.object(self.logger, 'info') as mock_logger:
            g = generate_coverage.GenerateFirstThenCollect(
                instance=instance,
                output_dir=self.temp_folder,
                cex_count=cex_count,
                spec=self.default_spec,
                heap_size=None,
                timelimit=self.default_timelimit,
                logger=self.logger,
                aa_file=aa_file,
                start_time=self.start_time,
                timer=generate_coverage.Timer())
            cex_generated = len(list(g.generate_executions()))
            mock_logger.assert_called_once_with('Generated 1 executions.')

        self.assertTrue(os.path.exists(self.temp_folder))
        self.assertEqual(len(os.listdir(self.temp_folder)), 1)
        self.assertEqual(cex_generated, 1)
Exemplo n.º 17
0
    def test(self):
        instance = os.path.join(self.aux_root, "three_paths.c")
        cex_count = 10  # There are only 3 paths though, checking the output.
        aa_file = os.path.join(self.aux_root, "dummy_aa.spc")
        with patch.object(self.logger, "info") as mock_info:
            g = generate_coverage.GenerateFirstThenCollect(
                instance=instance,
                output_dir=self.temp_folder,
                cex_count=cex_count,
                spec=self.default_spec,
                heap_size=None,
                timelimit=self.default_timelimit,
                logger=self.logger,
                aa_file=aa_file,
                start_time=self.start_time,
                timer=generate_coverage.Timer(),
            )
            cex_generated = len(list(g.generate_executions()))
            mock_info.assert_called_once_with("Generated 3 executions.")

        self.assertTrue(os.path.exists(self.temp_folder))
        self.assertEqual(len(os.listdir(self.temp_folder)), 3)
        self.assertEqual(cex_generated, 3)
Exemplo n.º 18
0
 def test(self):
     instance = os.path.join(self.aux_root, "three_paths.c")
     aa_file = os.path.join(self.aux_root, "single_true_state.spc")
     specs_dir = self.temp_folder
     with patch.object(self.logger, "info") as mock_info:
         g = generate_coverage.FixPointOnCoveredLines(
             instance=instance,
             output_dir=specs_dir,
             cex_count=1,
             spec=self.default_spec,
             heap_size=None,
             timelimit=None,
             logger=self.logger,
             aa_file=aa_file,
             start_time=self.start_time,
             timer=generate_coverage.Timer(),
         )
         cex_generated = list(g.generate_executions())
         expected_calls = [call("Generated 1 executions.")]
         self.assertEqual(mock_info.mock_calls, expected_calls)
     self.assertTrue(os.path.exists(self.temp_folder))
     self.assertEqual(len(os.listdir(self.temp_folder)), 2)
     self.assertEqual(len(cex_generated), 1)