def collect_testcases(self, module_path: Union[str, os.PathLike]) -> None:
"""Collect all test cases from a module.
Args:
module_path: Path to the module to collect the test cases from
"""
tree = self.get_ast_tree(module_path)
if tree is None:
config.configuration.initial_population_seeding = False
self._logger.info("Provided testcases are not used.")
return
transformer = _TestTransformer(self._test_cluster)
transformer.visit(tree)
self._testcases = transformer.testcases
if not self._testcases:
config.configuration.initial_population_seeding = False
self._logger.info("None of the provided test cases can be parsed.")
else:
self._logger.info(
"Number successfully collected test cases: %s", len(self._testcases)
)
stat.track_output_variable(
RuntimeVariable.CollectedTestCases, len(self._testcases)
)
self._mutate_testcases_initially()
def generate_tests(self) -> chrom.Chromosome:
self._logger.info("Start generating tests")
self._archive = Archive(set(self._fitness_functions))
self._number_of_goals = len(self._fitness_functions)
stat.set_output_variable_for_runtime_variable(RuntimeVariable.Goals,
self._number_of_goals)
self._current_iteration = 0
self._population = self._get_random_population()
self._archive.update(self._population)
# Calculate dominance ranks and crowding distance
fronts = self._ranking_function.compute_ranking_assignment(
self._population, self._archive.uncovered_goals)
for i in range(fronts.get_number_of_sub_fronts()):
fast_epsilon_dominance_assignment(fronts.get_sub_front(i),
self._archive.uncovered_goals)
while (not self._stopping_condition.is_fulfilled() and
self._number_of_goals - len(self._archive.covered_goals) != 0):
self.evolve()
self._notify_iteration()
self._current_iteration += 1
stat.track_output_variable(RuntimeVariable.AlgorithmIterations,
self._current_iteration)
return self.create_test_suite(self._archive.solutions if len(
self._archive.solutions) > 0 else self._get_best_individuals())
def test_variables_generator():
value_1 = MagicMock(Timer)
value_2 = MagicMock(Timer)
stat.track_output_variable(RuntimeVariable.TotalTime, value_1)
stat.track_output_variable(RuntimeVariable.TotalTime, value_2)
result = [v for _, v in stat.variables_generator]
assert result in ([], [value_1, value_2])
def _collect_statistics() -> None:
stat.track_output_variable(RuntimeVariable.TargetModule,
config.configuration.module_name)
stat.track_output_variable(RuntimeVariable.RandomSeed,
randomness.RNG.get_seed())
stat.track_output_variable(RuntimeVariable.ConfigurationId,
config.configuration.configuration_id)
stat.track_output_variable(RuntimeVariable.ProjectName,
config.configuration.project_name)
for runtime_variable, value in stat.variables_generator:
stat.set_output_variable_for_runtime_variable(runtime_variable, value)
def generate_tests(self, ) -> tsc.TestSuiteChromosome:
test_chromosome: tsc.TestSuiteChromosome = tsc.TestSuiteChromosome()
failing_test_chromosome: tsc.TestSuiteChromosome = tsc.TestSuiteChromosome(
)
generation: int = 0
for fitness_function in self._fitness_functions:
test_chromosome.add_fitness_function(fitness_function)
failing_test_chromosome.add_fitness_function(fitness_function)
combined_chromosome = self._combine_current_individual(
test_chromosome, failing_test_chromosome)
while (not self._stopping_condition.is_fulfilled()
and combined_chromosome.get_fitness() != 0.0):
try:
generation += 1
self._stopping_condition.iterate()
self.generate_sequence(
test_chromosome,
failing_test_chromosome,
generation,
)
combined_chromosome = self._combine_current_individual(
test_chromosome, failing_test_chromosome)
stat.current_individual(combined_chromosome)
self._logger.info(
"Generation: %5i. Best fitness: %5f, Best coverage %5f",
generation,
combined_chromosome.get_fitness(),
combined_chromosome.get_coverage(),
)
except (ConstructionFailedException,
GenerationException) as exception:
self._logger.debug(
"Generate test case failed with exception %s", exception)
self._logger.debug("Number of algorithm iterations: %d", generation)
stat.track_output_variable(RuntimeVariable.AlgorithmIterations,
generation)
combined_chromosome = self._combine_current_individual(
test_chromosome, failing_test_chromosome)
return combined_chromosome
def generate_tests(self, ) -> tsc.TestSuiteChromosome:
solution = self._get_random_solution()
stat.current_individual(solution)
generation = 0
while (not self._stopping_condition.is_fulfilled()
and solution.get_fitness() != 0.0):
candidate = self._get_random_solution()
if candidate.get_fitness() < solution.get_fitness():
solution = candidate
self._logger.info(
"Generation: %5i. Best fitness: %5f, Best coverage %5f",
generation,
solution.get_fitness(),
solution.get_coverage(),
)
stat.current_individual(solution)
generation += 1
stat.track_output_variable(RuntimeVariable.AlgorithmIterations,
generation)
return solution
def get_ast_tree(
self, module_path: Union[str, os.PathLike]
) -> Optional[ast.Module]:
"""Returns the ast tree from a module
Args:
module_path: The path to the project's root
Returns:
The ast tree of the given module.
"""
module_name = config.configuration.module_name.split(".")[-1]
self._logger.debug("Module name: %s", module_name)
result: List[str] = []
for root, _, files in os.walk(module_path):
for name in files:
if module_name in name and "test_" in name:
result.append(os.path.join(root, name))
break
try:
if len(result) > 0:
self._logger.debug("Module name found: %s", result[0])
stat.track_output_variable(RuntimeVariable.SuitableTestModule, True)
with open(result[0]) as module_file:
return ast.parse(module_file.read())
else:
self._logger.debug("No suitable test module found.")
stat.track_output_variable(RuntimeVariable.SuitableTestModule, False)
return None
except BaseException as exception: # pylint: disable=broad-except
self._logger.exception("Cannot read module: %s", exception)
stat.track_output_variable(RuntimeVariable.SuitableTestModule, False)
return None
def generate_tests(self, ) -> tsc.TestSuiteChromosome:
self._population = self._get_random_population()
self._sort_population()
stat.current_individual(self._get_best_individual())
generation = 0
while (not self._stopping_condition.is_fulfilled()
and self._get_best_individual().get_fitness() != 0.0):
self.evolve()
stat.current_individual(self._get_best_individual())
self._logger.info(
"Generation: %5i. Best fitness: %5f, Best coverage %5f",
generation,
self._get_best_individual().get_fitness(),
self._get_best_individual().get_coverage(),
)
generation += 1
stat.track_output_variable(RuntimeVariable.AlgorithmIterations,
generation)
best = self._get_best_individual()
# Make sure all test cases have a cached result.
best.get_fitness()
return best
def _track_sut_data(tracer: ExecutionTracer,
test_cluster: TestCluster) -> None:
"""Track data from the SUT.
Args:
tracer: the execution tracer
test_cluster: the test cluster
"""
stat.track_output_variable(
RuntimeVariable.CodeObjects,
len(tracer.get_known_data().existing_code_objects),
)
stat.track_output_variable(
RuntimeVariable.Predicates,
len(tracer.get_known_data().existing_predicates),
)
stat.track_output_variable(
RuntimeVariable.AccessibleObjectsUnderTest,
test_cluster.num_accessible_objects_under_test(),
)
stat.track_output_variable(
RuntimeVariable.GeneratableTypes,
len(test_cluster.get_all_generatable_types()),
)
def _track_statistics(
passing: chrom.Chromosome,
failing: chrom.Chromosome,
result: chrom.Chromosome,
) -> None:
stat.current_individual(result)
stat.track_output_variable(RuntimeVariable.Size, result.size())
stat.track_output_variable(RuntimeVariable.Length, result.length())
stat.track_output_variable(RuntimeVariable.FailingSize, failing.size())
stat.track_output_variable(
RuntimeVariable.FailingLength,
failing.length(),
)
stat.track_output_variable(RuntimeVariable.PassingSize, passing.size())
stat.track_output_variable(RuntimeVariable.PassingLength, passing.length())
return ReturnCode.SETUP_FAILED
executor, test_cluster = setup_result
with Timer(name="Test generation time", logger=None):
algorithm: TestGenerationStrategy = _instantiate_test_generation_strategy(
executor, test_cluster)
_LOGGER.info("Start generating sequences using %s",
config.configuration.algorithm)
stat.set_sequence_start_time(time.time_ns())
generation_result = algorithm.generate_tests()
_LOGGER.info("Stop generating sequences using %s",
config.configuration.algorithm)
algorithm.send_statistics()
with Timer(name="Re-execution time", logger=None):
stat.track_output_variable(RuntimeVariable.Coverage,
generation_result.get_coverage())
if config.configuration.post_process:
postprocessor = pp.ExceptionTruncation()
generation_result.accept(postprocessor)
# TODO(fk) add more postprocessing stuff.
if config.configuration.generate_assertions:
generator = ag.AssertionGenerator(executor)
generation_result.accept(generator)
with Timer(name="Export time", logger=None):
converter = cc.ChromosomeConverter()
generation_result.accept(converter)
failing = converter.failing_test_suite
passing = converter.passing_test_suite
def send_statistics(self) -> None:
super().send_statistics()
stat.track_output_variable(RuntimeVariable.ExecutionResults,
self._execution_results)
def visit_Module(self, node: ast.Module) -> Any:
self.generic_visit(node)
stat.track_output_variable(
RuntimeVariable.FoundTestCases, self._number_found_testcases
)