def collect_testcases(self, module_path: Union[str, os.PathLike]) -> None: """Collect all test cases from a module. Args: module_path: Path to the module to collect the test cases from """ tree = self.get_ast_tree(module_path) if tree is None: config.configuration.initial_population_seeding = False self._logger.info("Provided testcases are not used.") return transformer = _TestTransformer(self._test_cluster) transformer.visit(tree) self._testcases = transformer.testcases if not self._testcases: config.configuration.initial_population_seeding = False self._logger.info("None of the provided test cases can be parsed.") else: self._logger.info( "Number successfully collected test cases: %s", len(self._testcases) ) stat.track_output_variable( RuntimeVariable.CollectedTestCases, len(self._testcases) ) self._mutate_testcases_initially()
def generate_tests(self) -> chrom.Chromosome: self._logger.info("Start generating tests") self._archive = Archive(set(self._fitness_functions)) self._number_of_goals = len(self._fitness_functions) stat.set_output_variable_for_runtime_variable(RuntimeVariable.Goals, self._number_of_goals) self._current_iteration = 0 self._population = self._get_random_population() self._archive.update(self._population) # Calculate dominance ranks and crowding distance fronts = self._ranking_function.compute_ranking_assignment( self._population, self._archive.uncovered_goals) for i in range(fronts.get_number_of_sub_fronts()): fast_epsilon_dominance_assignment(fronts.get_sub_front(i), self._archive.uncovered_goals) while (not self._stopping_condition.is_fulfilled() and self._number_of_goals - len(self._archive.covered_goals) != 0): self.evolve() self._notify_iteration() self._current_iteration += 1 stat.track_output_variable(RuntimeVariable.AlgorithmIterations, self._current_iteration) return self.create_test_suite(self._archive.solutions if len( self._archive.solutions) > 0 else self._get_best_individuals())
def test_variables_generator(): value_1 = MagicMock(Timer) value_2 = MagicMock(Timer) stat.track_output_variable(RuntimeVariable.TotalTime, value_1) stat.track_output_variable(RuntimeVariable.TotalTime, value_2) result = [v for _, v in stat.variables_generator] assert result in ([], [value_1, value_2])
def _collect_statistics() -> None: stat.track_output_variable(RuntimeVariable.TargetModule, config.configuration.module_name) stat.track_output_variable(RuntimeVariable.RandomSeed, randomness.RNG.get_seed()) stat.track_output_variable(RuntimeVariable.ConfigurationId, config.configuration.configuration_id) stat.track_output_variable(RuntimeVariable.ProjectName, config.configuration.project_name) for runtime_variable, value in stat.variables_generator: stat.set_output_variable_for_runtime_variable(runtime_variable, value)
def generate_tests(self, ) -> tsc.TestSuiteChromosome: test_chromosome: tsc.TestSuiteChromosome = tsc.TestSuiteChromosome() failing_test_chromosome: tsc.TestSuiteChromosome = tsc.TestSuiteChromosome( ) generation: int = 0 for fitness_function in self._fitness_functions: test_chromosome.add_fitness_function(fitness_function) failing_test_chromosome.add_fitness_function(fitness_function) combined_chromosome = self._combine_current_individual( test_chromosome, failing_test_chromosome) while (not self._stopping_condition.is_fulfilled() and combined_chromosome.get_fitness() != 0.0): try: generation += 1 self._stopping_condition.iterate() self.generate_sequence( test_chromosome, failing_test_chromosome, generation, ) combined_chromosome = self._combine_current_individual( test_chromosome, failing_test_chromosome) stat.current_individual(combined_chromosome) self._logger.info( "Generation: %5i. Best fitness: %5f, Best coverage %5f", generation, combined_chromosome.get_fitness(), combined_chromosome.get_coverage(), ) except (ConstructionFailedException, GenerationException) as exception: self._logger.debug( "Generate test case failed with exception %s", exception) self._logger.debug("Number of algorithm iterations: %d", generation) stat.track_output_variable(RuntimeVariable.AlgorithmIterations, generation) combined_chromosome = self._combine_current_individual( test_chromosome, failing_test_chromosome) return combined_chromosome
def generate_tests(self, ) -> tsc.TestSuiteChromosome: solution = self._get_random_solution() stat.current_individual(solution) generation = 0 while (not self._stopping_condition.is_fulfilled() and solution.get_fitness() != 0.0): candidate = self._get_random_solution() if candidate.get_fitness() < solution.get_fitness(): solution = candidate self._logger.info( "Generation: %5i. Best fitness: %5f, Best coverage %5f", generation, solution.get_fitness(), solution.get_coverage(), ) stat.current_individual(solution) generation += 1 stat.track_output_variable(RuntimeVariable.AlgorithmIterations, generation) return solution
def get_ast_tree( self, module_path: Union[str, os.PathLike] ) -> Optional[ast.Module]: """Returns the ast tree from a module Args: module_path: The path to the project's root Returns: The ast tree of the given module. """ module_name = config.configuration.module_name.split(".")[-1] self._logger.debug("Module name: %s", module_name) result: List[str] = [] for root, _, files in os.walk(module_path): for name in files: if module_name in name and "test_" in name: result.append(os.path.join(root, name)) break try: if len(result) > 0: self._logger.debug("Module name found: %s", result[0]) stat.track_output_variable(RuntimeVariable.SuitableTestModule, True) with open(result[0]) as module_file: return ast.parse(module_file.read()) else: self._logger.debug("No suitable test module found.") stat.track_output_variable(RuntimeVariable.SuitableTestModule, False) return None except BaseException as exception: # pylint: disable=broad-except self._logger.exception("Cannot read module: %s", exception) stat.track_output_variable(RuntimeVariable.SuitableTestModule, False) return None
def generate_tests(self, ) -> tsc.TestSuiteChromosome: self._population = self._get_random_population() self._sort_population() stat.current_individual(self._get_best_individual()) generation = 0 while (not self._stopping_condition.is_fulfilled() and self._get_best_individual().get_fitness() != 0.0): self.evolve() stat.current_individual(self._get_best_individual()) self._logger.info( "Generation: %5i. Best fitness: %5f, Best coverage %5f", generation, self._get_best_individual().get_fitness(), self._get_best_individual().get_coverage(), ) generation += 1 stat.track_output_variable(RuntimeVariable.AlgorithmIterations, generation) best = self._get_best_individual() # Make sure all test cases have a cached result. best.get_fitness() return best
def _track_sut_data(tracer: ExecutionTracer, test_cluster: TestCluster) -> None: """Track data from the SUT. Args: tracer: the execution tracer test_cluster: the test cluster """ stat.track_output_variable( RuntimeVariable.CodeObjects, len(tracer.get_known_data().existing_code_objects), ) stat.track_output_variable( RuntimeVariable.Predicates, len(tracer.get_known_data().existing_predicates), ) stat.track_output_variable( RuntimeVariable.AccessibleObjectsUnderTest, test_cluster.num_accessible_objects_under_test(), ) stat.track_output_variable( RuntimeVariable.GeneratableTypes, len(test_cluster.get_all_generatable_types()), )
def _track_statistics( passing: chrom.Chromosome, failing: chrom.Chromosome, result: chrom.Chromosome, ) -> None: stat.current_individual(result) stat.track_output_variable(RuntimeVariable.Size, result.size()) stat.track_output_variable(RuntimeVariable.Length, result.length()) stat.track_output_variable(RuntimeVariable.FailingSize, failing.size()) stat.track_output_variable( RuntimeVariable.FailingLength, failing.length(), ) stat.track_output_variable(RuntimeVariable.PassingSize, passing.size()) stat.track_output_variable(RuntimeVariable.PassingLength, passing.length())
return ReturnCode.SETUP_FAILED executor, test_cluster = setup_result with Timer(name="Test generation time", logger=None): algorithm: TestGenerationStrategy = _instantiate_test_generation_strategy( executor, test_cluster) _LOGGER.info("Start generating sequences using %s", config.configuration.algorithm) stat.set_sequence_start_time(time.time_ns()) generation_result = algorithm.generate_tests() _LOGGER.info("Stop generating sequences using %s", config.configuration.algorithm) algorithm.send_statistics() with Timer(name="Re-execution time", logger=None): stat.track_output_variable(RuntimeVariable.Coverage, generation_result.get_coverage()) if config.configuration.post_process: postprocessor = pp.ExceptionTruncation() generation_result.accept(postprocessor) # TODO(fk) add more postprocessing stuff. if config.configuration.generate_assertions: generator = ag.AssertionGenerator(executor) generation_result.accept(generator) with Timer(name="Export time", logger=None): converter = cc.ChromosomeConverter() generation_result.accept(converter) failing = converter.failing_test_suite passing = converter.passing_test_suite
def send_statistics(self) -> None: super().send_statistics() stat.track_output_variable(RuntimeVariable.ExecutionResults, self._execution_results)
def visit_Module(self, node: ast.Module) -> Any: self.generic_visit(node) stat.track_output_variable( RuntimeVariable.FoundTestCases, self._number_found_testcases )