def test_clear():
tracer = ExecutionTracer()
tracer.register_code_object(MagicMock(CodeObjectMetaData))
tracer.executed_code_object(0)
trace = tracer.get_trace()
tracer.clear_trace()
assert tracer.get_trace() != trace
def test_update_metrics_false_dist_min():
tracer = ExecutionTracer()
tracer.register_predicate(MagicMock(PredicateMetaData))
tracer.executed_compare_predicate(3, 1, 0, Compare.NE)
assert (0, 2) in tracer.get_trace().false_distances.items()
tracer.executed_compare_predicate(2, 1, 0, Compare.NE)
assert (0, 1) in tracer.get_trace().false_distances.items()
def test_update_metrics_true_dist_min():
tracer = ExecutionTracer()
tracer.register_predicate(MagicMock(PredicateMetaData))
tracer.executed_compare_predicate(5, 0, 0, Compare.EQ)
assert (0, 5) in tracer.get_trace().true_distances.items()
tracer.executed_compare_predicate(4, 0, 0, Compare.EQ)
assert (0, 4) in tracer.get_trace().true_distances.items()
def test_compute_fitness_values_no_branches():
module_name = "tests.fixtures.branchcoverage.nobranches"
tracer = ExecutionTracer()
tracer.current_thread_ident = threading.currentThread().ident
with install_import_hook(module_name, tracer):
module = importlib.import_module(module_name)
importlib.reload(module)
executor = TestCaseExecutor(tracer)
chromosome = _get_test_for_no_branches_fixture(module)
goals = bcf.BranchCoverageFactory(executor).get_coverage_goals()
goals_dict = {}
for goal in goals:
chromosome.add_fitness_function(goal)
goals_dict[tracer.get_known_data().existing_code_objects[
goal._goal.code_object_id].code_object.co_name] = goal
fitness = chromosome.get_fitness()
assert fitness == 1
assert chromosome.fitness_values[goals_dict["__init__"]].fitness == 0.0
assert chromosome.fitness_values[goals_dict["other"]].fitness == 1.0
assert chromosome.fitness_values[goals_dict["<module>"]].fitness == 0.0
assert chromosome.fitness_values[goals_dict["get_x"]].fitness == 0.0
assert chromosome.fitness_values[goals_dict["identity"]].fitness == 0.0
assert chromosome.fitness_values[
goals_dict["DummyClass"]].fitness == 0.0
def test_hook():
tracer = ExecutionTracer()
with install_import_hook("tests.fixtures.instrumentation.mixed", tracer):
module = importlib.import_module(
"tests.fixtures.instrumentation.mixed")
importlib.reload(module)
assert len(tracer.get_known_data().existing_code_objects) > 0
assert module.function(6) == 0
def test_bool_distance_false():
tracer = ExecutionTracer()
tracer.current_thread_ident = threading.currentThread().ident
tracer.register_predicate(MagicMock(code_object_id=0))
tracer.executed_bool_predicate(False, 0)
assert (0, 1.0) in tracer.get_trace().true_distances.items()
assert (0, 0.0) in tracer.get_trace().false_distances.items()
def test_cmp(cmp, val1, val2, true_dist, false_dist):
tracer = ExecutionTracer()
tracer.current_thread_ident = threading.currentThread().ident
tracer.register_predicate(MagicMock(code_object_id=0))
tracer.executed_compare_predicate(val1, val2, 0, cmp)
assert (0, true_dist) in tracer.get_trace().true_distances.items()
assert (0, false_dist) in tracer.get_trace().false_distances.items()
def test_calculate_control_flow_distance_for_root(executed_code_objects,
approach_level):
execution_result = MagicMock(ExecutionResult)
execution_trace = MagicMock(ExecutionTrace)
execution_trace.executed_code_objects = executed_code_objects
execution_result.execution_trace = execution_trace
tracer = ExecutionTracer()
tracer.register_code_object(MagicMock())
tracer.register_code_object(MagicMock())
distance = get_root_control_flow_distance(execution_result, 0, tracer)
assert distance == ControlFlowDistance(approach_level=approach_level,
branch_distance=0.0)
def test_update_metrics_covered():
tracer = ExecutionTracer()
tracer.current_thread_ident = threading.currentThread().ident
tracer.register_predicate(MagicMock(code_object_id=0))
tracer.executed_compare_predicate(1, 0, 0, Compare.EQ)
tracer.executed_compare_predicate(1, 0, 0, Compare.EQ)
assert (0, 2) in tracer.get_trace().executed_predicates.items()
def _track_sut_data(tracer: ExecutionTracer, test_cluster: TestCluster) -> None:
"""Track data from the SUT."""
tracker = StatisticsTracker()
tracker.track_output_variable(
RuntimeVariable.CodeObjects,
len(tracer.get_known_data().existing_code_objects),
)
tracker.track_output_variable(
RuntimeVariable.Predicates, len(tracer.get_known_data().existing_predicates)
)
tracker.track_output_variable(
RuntimeVariable.AccessibleObjectsUnderTest,
test_cluster.num_accessible_objects_under_test(),
)
def test_compute_fitness_values_nested_branches():
module_name = "tests.fixtures.branchcoverage.nestedbranches"
tracer = ExecutionTracer()
tracer.current_thread_ident = threading.currentThread().ident
with install_import_hook(module_name, tracer):
module = importlib.import_module(module_name)
importlib.reload(module)
executor = TestCaseExecutor(tracer)
chromosome = _get_test_for_nested_branch_fixture(module)
goals = bcf.BranchCoverageFactory(executor).get_coverage_goals()
for goal in goals:
chromosome.add_fitness_function(goal)
fitness = chromosome.get_fitness()
assert fitness == pytest.approx(5.90782493)
def test_integrate_branch_distance_instrumentation(
simple_module,
function_name,
branchless_function_count,
branches_count,
):
tracer = ExecutionTracer()
function_callable = getattr(simple_module, function_name)
instr = BranchDistanceInstrumentation(tracer)
function_callable.__code__ = instr._instrument_code_recursive(
function_callable.__code__, 0)
assert (len(tracer.get_known_data().branch_less_code_objects) ==
branchless_function_count)
assert len(list(
tracer.get_known_data().existing_predicates)) == branches_count
def test_no_exceptions(short_test_case):
config.INSTANCE.module_name = "tests.fixtures.accessibles.accessible"
tracer = ExecutionTracer()
with install_import_hook(config.INSTANCE.module_name, tracer):
executor = TestCaseExecutor(tracer)
result = executor.execute([short_test_case])
assert not result.has_test_exceptions()
def test_module_instrumentation_integration():
"""Small integration test, which tests the instrumentation for various function types."""
tracer = ExecutionTracer()
with install_import_hook("tests.fixtures.instrumentation.mixed", tracer):
mixed = importlib.import_module("tests.fixtures.instrumentation.mixed")
mixed = importlib.reload(mixed)
inst = mixed.TestClass(5)
inst.method(5)
inst.method_with_nested(5)
mixed.function(5)
sum(mixed.generator())
asyncio.run(mixed.coroutine(5))
asyncio.run(run_async_generator(mixed.async_generator()))
assert len(tracer.get_trace().executed_code_objects) == 10
def test_bool_predicate_executed_other_thread():
tracer = ExecutionTracer()
tracer.current_thread_ident = threading.currentThread().ident
tracer.register_code_object(MagicMock())
tracer.register_code_object(MagicMock(code_object_id=0))
thread = threading.Thread(target=tracer.executed_bool_predicate, args=(True, 0))
thread.start()
thread.join()
assert tracer.get_trace().executed_predicates == {}
def test_simple_execution():
config.INSTANCE.module_name = "tests.fixtures.accessibles.accessible"
tracer = ExecutionTracer()
with install_import_hook(config.INSTANCE.module_name, tracer):
test_case = dtc.DefaultTestCase()
test_case.add_statement(prim_stmt.IntPrimitiveStatement(test_case, 5))
executor = TestCaseExecutor(tracer)
assert not executor.execute([test_case]).has_test_exceptions()
def test_no_branchless_code_object_register_multiple():
tracer = ExecutionTracer()
tracer.register_code_object(MagicMock())
tracer.register_code_object(MagicMock())
tracer.register_predicate(MagicMock(code_object_id=0))
tracer.register_predicate(MagicMock(code_object_id=0))
assert tracer.get_known_data().branch_less_code_objects == {1}
def test_no_exceptions(short_test_case):
config.configuration.module_name = "tests.fixtures.accessibles.accessible"
tracer = ExecutionTracer()
with install_import_hook(config.configuration.module_name, tracer):
module = importlib.import_module(config.configuration.module_name)
importlib.reload(module)
executor = TestCaseExecutor(tracer)
result = executor.execute(short_test_case)
assert not result.has_test_exceptions()
def test_simple_execution():
config.configuration.module_name = "tests.fixtures.accessibles.accessible"
tracer = ExecutionTracer()
with install_import_hook(config.configuration.module_name, tracer):
module = importlib.import_module(config.configuration.module_name)
importlib.reload(module)
test_case = dtc.DefaultTestCase()
test_case.add_statement(prim_stmt.IntPrimitiveStatement(test_case, 5))
executor = TestCaseExecutor(tracer)
assert not executor.execute(test_case).has_test_exceptions()
def test_observers(short_test_case):
tracer = ExecutionTracer()
executor = TestCaseExecutor(tracer)
observer = MagicMock()
executor.add_observer(observer)
executor.execute(short_test_case)
assert observer.before_test_case_execution.call_count == 1
assert observer.before_statement_execution.call_count == 2
assert observer.after_statement_execution.call_count == 2
assert observer.after_test_case_execution.call_count == 1
def _load_sut(tracer: ExecutionTracer) -> bool:
try:
# We need to set the current thread ident so the import trace is recorded.
tracer.current_thread_ident = threading.currentThread().ident
importlib.import_module(config.configuration.module_name)
except ImportError as ex:
# A module could not be imported because some dependencies
# are missing or it is malformed
_LOGGER.exception("Failed to load SUT: %s", ex)
return False
return True
def test_illegal_call(method_mock):
config.INSTANCE.module_name = "tests.fixtures.accessibles.accessible"
test_case = dtc.DefaultTestCase()
int_stmt = prim_stmt.IntPrimitiveStatement(test_case, 5)
method_stmt = param_stmt.MethodStatement(test_case, method_mock,
int_stmt.return_value)
test_case.add_statement(int_stmt)
test_case.add_statement(method_stmt)
tracer = ExecutionTracer()
with install_import_hook(config.INSTANCE.module_name, tracer):
executor = TestCaseExecutor(tracer)
result = executor.execute([test_case])
assert result.has_test_exceptions()
def test_code_object_executed_other_thread():
tracer = ExecutionTracer()
tracer.current_thread_ident = threading.currentThread().ident
tracer.register_code_object(MagicMock())
thread = threading.Thread(target=tracer.executed_code_object, args=(0,))
thread.start()
thread.join()
assert tracer.get_trace().executed_code_objects == set()
def _track_sut_data(tracer: ExecutionTracer,
test_cluster: TestCluster) -> None:
"""Track data from the SUT.
Args:
tracer: the execution tracer
test_cluster: the test cluster
"""
stat.track_output_variable(
RuntimeVariable.CodeObjects,
len(tracer.get_known_data().existing_code_objects),
)
stat.track_output_variable(
RuntimeVariable.Predicates,
len(tracer.get_known_data().existing_predicates),
)
stat.track_output_variable(
RuntimeVariable.AccessibleObjectsUnderTest,
test_cluster.num_accessible_objects_under_test(),
)
stat.track_output_variable(
RuntimeVariable.GeneratableTypes,
len(test_cluster.get_all_generatable_types()),
)
def test_illegal_call(method_mock):
config.configuration.module_name = "tests.fixtures.accessibles.accessible"
test_case = dtc.DefaultTestCase()
int_stmt = prim_stmt.IntPrimitiveStatement(test_case, 5)
method_stmt = param_stmt.MethodStatement(test_case, method_mock,
int_stmt.ret_val)
test_case.add_statement(int_stmt)
test_case.add_statement(method_stmt)
tracer = ExecutionTracer()
with install_import_hook(config.configuration.module_name, tracer):
module = importlib.import_module(config.configuration.module_name)
importlib.reload(module)
executor = TestCaseExecutor(tracer)
result = executor.execute(test_case)
assert result.has_test_exceptions()
def _setup_path_and_hook(self) -> Optional[ExecutionTracer]:
"""Inserts the path to the SUT into the path list.
Also installs the import hook."""
if not os.path.isdir(config.INSTANCE.project_path):
self._logger.error(
"%s is not a valid project path", config.INSTANCE.project_path
)
return None
self._logger.debug("Setting up path for %s", config.INSTANCE.project_path)
sys.path.insert(0, config.INSTANCE.project_path)
self._logger.debug(
"Setting up instrumentation for %s", config.INSTANCE.module_name
)
tracer = ExecutionTracer()
install_import_hook(config.INSTANCE.module_name, tracer)
return tracer
def test_integrate_randoopy(module_name: str):
config.configuration.budget = 1
config.configuration.algorithm = config.Algorithm.RANDOM
config.configuration.module_name = module_name
logger = MagicMock(Logger)
tracer = ExecutionTracer()
with install_import_hook(module_name, tracer):
# Need to force reload in order to apply instrumentation.
module = importlib.import_module(module_name)
importlib.reload(module)
executor = TestCaseExecutor(tracer)
cluster = TestClusterGenerator(module_name).generate_cluster()
algorithm = gaf.TestSuiteGenerationAlgorithmFactory(
executor, cluster).get_search_algorithm()
algorithm._logger = logger
test_cases = algorithm.generate_tests()
assert test_cases.size() >= 0
def test_integrate_randoopy(algorithm_to_run: Callable, module_name: str):
config.INSTANCE.budget = 1
config.INSTANCE.module_name = module_name
logger = MagicMock(Logger)
tracer = ExecutionTracer()
with install_import_hook(module_name, tracer):
# Need to force reload in order to apply instrumentation.
module = importlib.import_module(module_name)
importlib.reload(module)
executor = TestCaseExecutor(tracer)
algorithm = algorithm_to_run(
executor,
TestClusterGenerator(module_name).generate_cluster())
algorithm._logger = logger
test_cases, failing_test_cases = algorithm.generate_sequences()
assert test_cases.size() >= 0
assert failing_test_cases.size() >= 0
def test_integrate_randomsearch(module_name: str):
# TODO(fk) reduce direct dependencies to config.INSTANCE
config.configuration.algorithm = config.Algorithm.RANDOM_SEARCH
config.configuration.budget = 1
config.configuration.module_name = module_name
config.configuration.min_initial_tests = 1
config.configuration.max_initial_tests = 1
logger = MagicMock(Logger)
tracer = ExecutionTracer()
with install_import_hook(module_name, tracer):
# Need to force reload in order to apply instrumentation.
module = importlib.import_module(module_name)
importlib.reload(module)
executor = TestCaseExecutor(tracer)
cluster = TestClusterGenerator(module_name).generate_cluster()
algorithm = gaf.TestSuiteGenerationAlgorithmFactory(
executor, cluster
).get_search_algorithm()
algorithm._logger = logger
test_cases = algorithm.generate_tests()
assert test_cases.size() >= 0
def test_integrate_wspy(module_name: str):
# TODO(fk) reduce direct dependencies to config.INSTANCE
config.INSTANCE.budget = 1
config.INSTANCE.module_name = module_name
config.INSTANCE.population = 3
config.INSTANCE.min_initial_tests = 1
config.INSTANCE.max_initial_tests = 1
logger = MagicMock(Logger)
tracer = ExecutionTracer()
with install_import_hook(module_name, tracer):
# Need to force reload in order to apply instrumentation.
module = importlib.import_module(module_name)
importlib.reload(module)
executor = TestCaseExecutor(tracer)
algorithm = WholeSuiteTestStrategy(
executor, TestClusterGenerator(module_name).generate_cluster()
)
algorithm._logger = logger
test_cases, failing_test_cases = algorithm.generate_sequences()
assert test_cases.size() >= 0
assert failing_test_cases.size() >= 0
