def test_create_mutant_with_cache(binop_file, stdoutIO):
"""Change ast.Add to ast.Mult in a mutation including pycache changes."""
genome = Genome(source_file=binop_file)
# this target is the add_five() function, changing add to mult
end_lineno = None if sys.version_info < (3, 8) else 10
end_col_offset = None if sys.version_info < (3, 8) else 16
target_idx = LocIndex(
ast_class="BinOp",
lineno=10,
col_offset=11,
op_type=ast.Add,
end_lineno=end_lineno,
end_col_offset=end_col_offset,
)
mutation_op = ast.Mult
mutant = genome.mutate(target_idx, mutation_op, write_cache=True)
# uses the redirection for stdout to capture the value from the final output of binop_file
with stdoutIO() as s:
exec(mutant.mutant_code)
assert int(s.getvalue()) == 25
tag = sys.implementation.cache_tag
expected_cfile = binop_file.parent / "__pycache__" / ".".join([binop_file.stem, tag, "pyc"])
assert mutant.src_file == binop_file
assert mutant.cfile == expected_cfile
assert mutant.src_idx == target_idx
def test_mutate_TypeError_source_file(mock_LocIdx):
"""Mutate with a NoneType source_file property raises a TypeError."""
genome = Genome()
with pytest.raises(TypeError):
_ = genome.mutate(target_idx=mock_LocIdx,
mutation_op=ast.Div,
write_cache=False)
def test_mutate_MutationException(binop_file, mock_LocIdx):
"""Mutate with an invalid operation raises a mutation exception."""
genome = Genome(binop_file)
with pytest.raises(MutationException):
_ = genome.mutate(target_idx=mock_LocIdx,
mutation_op="badoperation",
write_cache=False)
def test_mutate_ValueError_target(binop_file, mock_LocIdx):
"""Mutate with a target_idx not in the targets raises a ValueError."""
genome = Genome(binop_file)
with pytest.raises(ValueError):
_ = genome.mutate(target_idx=mock_LocIdx,
mutation_op=ast.Div,
write_cache=False)
def test_covered_targets(filter_codes, binop_file, mock_binop_coverage_file):
"""Mock coverage file sets lines 6 and 10 (not 15) to be covered."""
genome = Genome(binop_file, coverage_file=mock_binop_coverage_file)
genome.filter_codes = filter_codes
assert len(genome.targets) == 4
assert len(genome.covered_targets) == 3
for ct in genome.covered_targets:
assert ct.lineno in [6, 10]
diff = list(genome.targets - genome.covered_targets)
assert diff[0].lineno == 15
def add_five_to_mult_mutant(binop_file, stdoutIO, binop_Add_LocIdx):
"""Mutant that takes add_five op ADD to MULT. Fails if mutation code does not work."""
genome = Genome(source_file=binop_file)
mutation_op = ast.Mult
mutant = genome.mutate(binop_Add_LocIdx, mutation_op, write_cache=True)
# uses the redirection for stdout to capture the value from the final output of binop_file
with stdoutIO() as s:
exec(mutant.mutant_code)
assert int(s.getvalue()) == 25
return mutant
def test_MutateAST_visit_binop_37(binop_file):
"""Read only test to ensure locations are aggregated."""
tree = Genome(binop_file).ast
# Py 3.7 vs. Py 3.8
end_lineno = None if sys.version_info < (3, 8) else 6
end_col_offset = None if sys.version_info < (3, 8) else 17
test_idx = LocIndex(
ast_class="BinOp",
lineno=6,
col_offset=11,
op_type=ast.Add,
end_lineno=end_lineno,
end_col_offset=end_col_offset,
)
test_mutation = ast.Pow
# apply the mutation to the original tree copy
testing_tree = deepcopy(tree)
mutated_tree = MutateAST(target_idx=test_idx,
mutation=test_mutation).visit(testing_tree)
# revisit in read-only mode to gather the locations of the new nodes
mast = MutateAST(readonly=True)
mast.visit(mutated_tree)
# four locations from the binary operations in binop_file
assert len(mast.locs) == 4
# locs is an unordered set, cycle through to thd target and check the mutation
for l in mast.locs:
if (l.lineno == 6 and l.col_offset == 11 and l.end_lineno == end_lineno
and l.end_col_offset == end_col_offset):
assert l.op_type == test_mutation
def test_MutateAST_visit_if(if_file, if_expected_locs):
"""Test mutation for nameconst: True, False, None."""
tree = Genome(if_file).ast
test_mutation = "If_True"
testing_tree = deepcopy(tree)
# change from If_Statement to If_True
mutated_tree = MutateAST(target_idx=if_expected_locs[0],
mutation=test_mutation).visit(testing_tree)
mast = MutateAST(readonly=True)
mast.visit(mutated_tree)
# named constants will also be picked up, filter just to if_ operations
if_locs = [l for l in mast.locs if l.ast_class == "If"]
assert len(if_locs) == 4
for l in if_locs:
# spot check on mutation from True to False
if l.lineno == 2 and l.col_offset == 4:
print(l)
assert l.op_type == test_mutation
# spot check on not-mutated location still being None
if l.lineno == 13 and l.col_offset == 4:
assert l.op_type == "If_False"
def test_MutateAST_visit_nameconst(nameconst_file, nameconst_expected_locs):
"""Test mutation for nameconst: True, False, None."""
tree = Genome(nameconst_file).ast
test_mutation = False
testing_tree = deepcopy(tree)
mutated_tree = MutateAST(target_idx=nameconst_expected_locs[0], mutation=test_mutation).visit(
testing_tree
)
mast = MutateAST(readonly=True)
mast.visit(mutated_tree)
# if statement is included with this file that will be picked up
nc_locs = [loc for loc in mast.locs if loc.ast_class == "NameConstant"]
assert len(nc_locs) == 4
for loc in nc_locs:
# spot check on mutation from True to False
if loc.lineno == 1 and loc.col_offset == 14:
assert loc.op_type == test_mutation
# spot check on not-mutated location still being None
if loc.lineno == 7 and loc.col_offset == 22:
assert loc.op_type is None
def test_create_mutation_and_run_trial(returncode, expected_status,
monkeypatch, binop_file,
binop_Add_LocIdx):
"""Mocked trial to ensure mutated cache files are removed after running."""
genome = Genome(source_file=binop_file)
mutation_op = ast.Mult
tag = sys.implementation.cache_tag
expected_cfile = binop_file.parent / "__pycache__" / ".".join(
[binop_file.stem, tag, "pyc"])
def mock_subprocess_run(*args, **kwargs):
return CompletedProcess(args="pytest", returncode=returncode)
monkeypatch.setattr(subprocess, "run", mock_subprocess_run)
trial = run.create_mutation_run_trial(
genome=genome,
target_idx=binop_Add_LocIdx,
mutation_op=mutation_op,
test_cmds=["pytest"],
max_runtime=10,
)
# mutated cache files should be removed after trial run
assert not expected_cfile.exists()
assert trial.status == expected_status
def create_mutation_run_trial(genome: Genome, target_idx: LocIndex,
mutation_op: Any,
test_cmds: List[str]) -> MutantTrialResult:
"""Run a single mutation trial by creating a new mutated cache file, running the
test commands, and then removing the mutated cache file.
Args:
genome: the genome to mutate
target_idx: the mutation location
mutation_op: the mutation operation
test_cmds: the test commands to execute with the mutated code
Returns:
The mutation trial result
"""
LOGGER.debug("Running trial for %s", mutation_op)
mutant = genome.mutate(target_idx, mutation_op, write_cache=True)
mutant_trial = subprocess.run(test_cmds,
capture_output=capture_output(
LOGGER.getEffectiveLevel()))
cache.remove_existing_cache_files(mutant.src_file)
return MutantTrialResult(mutant=mutant,
return_code=mutant_trial.returncode)
def add_five_to_mult_mutant(binop_file, stdoutIO):
"""Mutant that takes add_five op ADD to MULT. Fails if mutation code does not work."""
genome = Genome(source_file=binop_file)
# this target is the add_five() function, changing add to mult
target_idx = LocIndex(ast_class="BinOp", lineno=10, col_offset=11, op_type=ast.Add)
mutation_op = ast.Mult
mutant = genome.mutate(target_idx, mutation_op, write_cache=True)
# uses the redirection for stdout to capture the value from the final output of binop_file
with stdoutIO() as s:
exec(mutant.mutant_code)
assert int(s.getvalue()) == 25
return mutant
def test_MutateAST_visit_index_neg(
i_order, lineno, col_offset, mut, index_file, index_expected_locs
):
"""Test mutation for Index: i[0], i[1], i[-1]."""
tree = Genome(index_file).ast
test_mutation = mut
testing_tree = deepcopy(tree)
mutated_tree = MutateAST(target_idx=index_expected_locs[i_order], mutation=test_mutation).visit(
testing_tree
)
mast = MutateAST(readonly=True)
mast.visit(mutated_tree)
assert len(mast.locs) == 4
for loc in mast.locs:
# spot check on mutation from Index_NumNeg to Index_NumPos
if loc.lineno == lineno and loc.col_offset == col_offset:
assert loc.op_type == test_mutation
# spot check on not-mutated location still being None
if loc.lineno == 4 and loc.col_offset == 23:
assert loc.op_type == "Index_NumPos"
def test_MutateAST_visit_read_only(binop_file):
"""Read only test to ensure locations are aggregated."""
tree = Genome(binop_file).ast
mast = MutateAST(readonly=True)
testing_tree = deepcopy(tree)
mast.visit(testing_tree)
# four locations from the binary operations in binop_file
assert len(mast.locs) == 4
# tree should be unmodified
assert ast.dump(tree) == ast.dump(testing_tree)
def test_MutateAST_visit_subscript(slice_file, slice_expected_locs):
"""Test Slice references within subscript."""
tree = Genome(slice_file).ast
mast = MutateAST(readonly=True)
mast.visit(tree)
assert len(mast.locs) == len(slice_expected_locs)
test_mutation = "Slice_UNegToZero"
mutated_tree = MutateAST(target_idx=slice_expected_locs[2], mutation=test_mutation).visit(tree)
mast.visit(mutated_tree)
assert len(mast.locs) == len(slice_expected_locs)
for loc in mast.locs:
if loc.lineno == 5 and loc.col_offset == 15:
assert loc.op_type == test_mutation
# test one unmodified location
if loc.lineno == 4 and loc.col_offset == 14:
assert loc.op_type == "Slice_UnboundUpper"
def test_MutateAST_visit_boolop(boolop_file, boolop_expected_loc):
"""Test mutation of AND to OR in the boolop."""
tree = Genome(boolop_file).ast
test_mutation = ast.Or
# apply the mutation to the original tree copy
testing_tree = deepcopy(tree)
mutated_tree = MutateAST(target_idx=boolop_expected_loc,
mutation=test_mutation).visit(testing_tree)
# revisit in read-only mode to gather the locations of the new nodes
mast = MutateAST(readonly=True)
mast.visit(mutated_tree)
# four locations from the binary operations in binop_file
assert len(mast.locs) == 1
# there will only be one loc, but this still works
# basedon the col and line offset in the fixture for compare_expected_loc
for l in mast.locs:
if l.lineno == 2 and l.col_offset == 11:
assert l.op_type == test_mutation
def test_MutateAST_visit_augassign(augassign_file, augassign_expected_locs):
"""Test mutation for AugAssign: +=, -=, /=, *=."""
tree = Genome(augassign_file).ast
test_mutation = "AugAssign_Div"
testing_tree = deepcopy(tree)
mutated_tree = MutateAST(target_idx=augassign_expected_locs[0],
mutation=test_mutation).visit(testing_tree)
mast = MutateAST(readonly=True)
mast.visit(mutated_tree)
assert len(mast.locs) == 4
for l in mast.locs:
# spot check on mutation from Add tp Div
if l.lineno == 1 and l.col_offset == 4:
assert l.op_type == test_mutation
# spot check on not-mutated location still being Mult
if l.lineno == 5 and l.col_offset == 4:
assert l.op_type == "AugAssign_Mult"
def create_mutation_run_trial(genome: Genome, target_idx: LocIndex,
mutation_op: Any, test_cmds: List[str],
max_runtime: float) -> MutantTrialResult:
"""Run a single mutation trial by creating a new mutated cache file, running the
test commands, and then removing the mutated cache file.
Args:
genome: the genome to mutate
target_idx: the mutation location
mutation_op: the mutation operation
test_cmds: the test commands to execute with the mutated code
max_runtime: timeout for the trial
Returns:
The mutation trial result
"""
LOGGER.debug("Running trial for %s", mutation_op)
mutant = genome.mutate(target_idx, mutation_op, write_cache=True)
try:
mutant_trial = subprocess.run(
test_cmds,
capture_output=capture_output(LOGGER.getEffectiveLevel()),
timeout=max_runtime,
)
return_code = mutant_trial.returncode
except subprocess.TimeoutExpired:
return_code = 3
cache.remove_existing_cache_files(mutant.src_file)
return MutantTrialResult(
mutant=MutantReport(src_file=mutant.src_file,
src_idx=mutant.src_idx,
mutation=mutant.mutation),
return_code=return_code,
)
def test_create_mutation_and_run_trial(returncode, expected_status, monkeypatch, binop_file):
"""Mocked trial to ensure mutated cache files are removed after running."""
genome = Genome(source_file=binop_file)
# this target is the add_five() function, changing add to mult
target_idx = LocIndex(ast_class="BinOp", lineno=10, col_offset=11, op_type=ast.Add)
mutation_op = ast.Mult
tag = sys.implementation.cache_tag
expected_cfile = binop_file.parent / "__pycache__" / ".".join([binop_file.stem, tag, "pyc"])
def mock_subprocess_run(*args, **kwargs):
return CompletedProcess(args="pytest", returncode=returncode)
monkeypatch.setattr(subprocess, "run", mock_subprocess_run)
trial = run.create_mutation_run_trial(
genome=genome, target_idx=target_idx, mutation_op=mutation_op, test_cmds=["pytest"]
)
# mutated cache files should be removed after trial run
assert not expected_cfile.exists()
assert trial.status == expected_status
def test_MutateAST_visit_compare(idx, mut_op, lineno, compare_file, compare_expected_locs):
"""Test mutation of the == to != in the compare op."""
tree = Genome(compare_file).ast
# apply the mutation to the original tree copy
testing_tree = deepcopy(tree)
mutated_tree = MutateAST(target_idx=compare_expected_locs[idx], mutation=mut_op).visit(
testing_tree
)
# revisit in read-only mode to gather the locations of the new nodes
mast = MutateAST(readonly=True)
mast.visit(mutated_tree)
assert len(mast.locs) == 3
# check that the lineno marked for mutation is changed, otherwise original ops should
# still be present without modification
for loc in mast.locs:
if loc.lineno == lineno and loc.col_offset == 11:
assert loc.op_type == mut_op
else:
assert loc.op_type in {ast.Eq, ast.Is, ast.In} # based on compare_file fixture
def test_covered_targets_source_file_TypeError():
"""Targets with a NoneType source_file raises a TypeError."""
with pytest.raises(TypeError):
genome = Genome()
_ = genome.covered_targets
def test_covered_targets_coverage_file_TypeError(binop_file):
"""Targets with a NoneType coverage_file but valid source_file raises a TypeError."""
with pytest.raises(TypeError):
genome = Genome(binop_file)
genome.coverage_file = None
_ = genome.covered_targets
def test_filter_codes_ValueError():
"""Setting invalid filter codes on the Genome raises a ValueError."""
with pytest.raises(ValueError):
genome = Genome()
genome.filter_codes = ("asdf",)
def test_GenomeGroup_TypeError_source_file():
"""GenomeGroup raises a TypeError adding a Genome without a set source_file."""
ggrp = GenomeGroup()
with pytest.raises(TypeError):
ggrp.add_genome(Genome())
def create_mutation_run_parallelcache_trial(
genome: Genome, target_idx: LocIndex, mutation_op: Any,
test_cmds: List[str], max_runtime: float) -> MutantTrialResult:
"""Similar to run.create_mutation_run_trial() but using the parallel cache directory settings.
This function requires Python 3.8 and does not run with Python 3.7. Importantly, it has the
identical signature to run.create_mutation_run_trial() and is substituted in the
run.mutation_sample_dispatch().
Args:
genome: the genome to mutate
target_idx: the mutation location
mutation_op: the mutation operation
test_cmds: the test commands to execute with the mutated code
max_runtime: timeout for the subprocess trial
Returns:
MutantTrialResult
Raises:
EnvironmentError: if Python version is less than 3.8
"""
if sys.version_info < (3, 8):
raise EnvironmentError(
"Python 3.8 is required to use PYTHONPYCACHEPREFIX.")
# Note in coverage reports this shows as untested code due to the subprocess dispatching
# the 'slow' tests in `test_run.py` cover this.
cache.check_cache_invalidation_mode()
# create the mutant without writing the cache
mutant = genome.mutate(target_idx, mutation_op, write_cache=False)
# set up parallel cache structure
parallel_cache = Path.cwd() / PARALLEL_PYCACHE_DIR / uuid.uuid4().hex
resolved_source_parts = genome.source_file.resolve().parent.parts[
1:] # type: ignore
parallel_cfile = parallel_cache.joinpath(
*resolved_source_parts) / mutant.cfile.name
bytecode = importlib._bootstrap_external._code_to_timestamp_pyc( # type: ignore
mutant.mutant_code, mutant.source_stats["mtime"],
mutant.source_stats["size"])
LOGGER.debug("Writing parallel mutant cache file: %s", parallel_cfile)
cache.create_cache_dirs(parallel_cfile)
importlib._bootstrap_external._write_atomic( # type: ignore
parallel_cfile,
bytecode,
mutant.mode,
)
copy_env = os.environ.copy()
copy_env["PYTHONPYCACHEPREFIX"] = str(parallel_cache)
try:
mutant_trial = subprocess.run(
test_cmds,
env=copy_env,
capture_output=capture_output(LOGGER.getEffectiveLevel()),
timeout=max_runtime + MULTI_PROC_TIMEOUT_BUFFER,
)
return_code = mutant_trial.returncode
except subprocess.TimeoutExpired:
return_code = 3
LOGGER.debug("Removing parallel cache file: %s", parallel_cache.parts[-1])
shutil.rmtree(parallel_cache)
return MutantTrialResult(
mutant=MutantReport(src_file=mutant.src_file,
src_idx=mutant.src_idx,
mutation=mutant.mutation),
return_code=return_code,
)
def test_GenomeGroup_key_TypeError(key, binop_file):
"""Values that are not Path type keys raise a type error."""
with pytest.raises(TypeError):
ggrp = GenomeGroup()
ggrp[key] = Genome(binop_file)
def test_genome_ast(binop_file, binop_expected_locs):
"""Test that the AST builds expected targets."""
genome = Genome(source_file=binop_file)
assert len(genome.targets) == 4
assert genome.targets == binop_expected_locs
