def test_create_mutant_with_cache(binop_file, stdoutIO): """Change ast.Add to ast.Mult in a mutation including pycache changes.""" genome = Genome(source_file=binop_file) # this target is the add_five() function, changing add to mult end_lineno = None if sys.version_info < (3, 8) else 10 end_col_offset = None if sys.version_info < (3, 8) else 16 target_idx = LocIndex( ast_class="BinOp", lineno=10, col_offset=11, op_type=ast.Add, end_lineno=end_lineno, end_col_offset=end_col_offset, ) mutation_op = ast.Mult mutant = genome.mutate(target_idx, mutation_op, write_cache=True) # uses the redirection for stdout to capture the value from the final output of binop_file with stdoutIO() as s: exec(mutant.mutant_code) assert int(s.getvalue()) == 25 tag = sys.implementation.cache_tag expected_cfile = binop_file.parent / "__pycache__" / ".".join([binop_file.stem, tag, "pyc"]) assert mutant.src_file == binop_file assert mutant.cfile == expected_cfile assert mutant.src_idx == target_idx
def test_mutate_TypeError_source_file(mock_LocIdx): """Mutate with a NoneType source_file property raises a TypeError.""" genome = Genome() with pytest.raises(TypeError): _ = genome.mutate(target_idx=mock_LocIdx, mutation_op=ast.Div, write_cache=False)
def test_mutate_MutationException(binop_file, mock_LocIdx): """Mutate with an invalid operation raises a mutation exception.""" genome = Genome(binop_file) with pytest.raises(MutationException): _ = genome.mutate(target_idx=mock_LocIdx, mutation_op="badoperation", write_cache=False)
def test_mutate_ValueError_target(binop_file, mock_LocIdx): """Mutate with a target_idx not in the targets raises a ValueError.""" genome = Genome(binop_file) with pytest.raises(ValueError): _ = genome.mutate(target_idx=mock_LocIdx, mutation_op=ast.Div, write_cache=False)
def test_covered_targets(filter_codes, binop_file, mock_binop_coverage_file): """Mock coverage file sets lines 6 and 10 (not 15) to be covered.""" genome = Genome(binop_file, coverage_file=mock_binop_coverage_file) genome.filter_codes = filter_codes assert len(genome.targets) == 4 assert len(genome.covered_targets) == 3 for ct in genome.covered_targets: assert ct.lineno in [6, 10] diff = list(genome.targets - genome.covered_targets) assert diff[0].lineno == 15
def add_five_to_mult_mutant(binop_file, stdoutIO, binop_Add_LocIdx): """Mutant that takes add_five op ADD to MULT. Fails if mutation code does not work.""" genome = Genome(source_file=binop_file) mutation_op = ast.Mult mutant = genome.mutate(binop_Add_LocIdx, mutation_op, write_cache=True) # uses the redirection for stdout to capture the value from the final output of binop_file with stdoutIO() as s: exec(mutant.mutant_code) assert int(s.getvalue()) == 25 return mutant
def test_MutateAST_visit_binop_37(binop_file): """Read only test to ensure locations are aggregated.""" tree = Genome(binop_file).ast # Py 3.7 vs. Py 3.8 end_lineno = None if sys.version_info < (3, 8) else 6 end_col_offset = None if sys.version_info < (3, 8) else 17 test_idx = LocIndex( ast_class="BinOp", lineno=6, col_offset=11, op_type=ast.Add, end_lineno=end_lineno, end_col_offset=end_col_offset, ) test_mutation = ast.Pow # apply the mutation to the original tree copy testing_tree = deepcopy(tree) mutated_tree = MutateAST(target_idx=test_idx, mutation=test_mutation).visit(testing_tree) # revisit in read-only mode to gather the locations of the new nodes mast = MutateAST(readonly=True) mast.visit(mutated_tree) # four locations from the binary operations in binop_file assert len(mast.locs) == 4 # locs is an unordered set, cycle through to thd target and check the mutation for l in mast.locs: if (l.lineno == 6 and l.col_offset == 11 and l.end_lineno == end_lineno and l.end_col_offset == end_col_offset): assert l.op_type == test_mutation
def test_MutateAST_visit_if(if_file, if_expected_locs): """Test mutation for nameconst: True, False, None.""" tree = Genome(if_file).ast test_mutation = "If_True" testing_tree = deepcopy(tree) # change from If_Statement to If_True mutated_tree = MutateAST(target_idx=if_expected_locs[0], mutation=test_mutation).visit(testing_tree) mast = MutateAST(readonly=True) mast.visit(mutated_tree) # named constants will also be picked up, filter just to if_ operations if_locs = [l for l in mast.locs if l.ast_class == "If"] assert len(if_locs) == 4 for l in if_locs: # spot check on mutation from True to False if l.lineno == 2 and l.col_offset == 4: print(l) assert l.op_type == test_mutation # spot check on not-mutated location still being None if l.lineno == 13 and l.col_offset == 4: assert l.op_type == "If_False"
def test_MutateAST_visit_nameconst(nameconst_file, nameconst_expected_locs): """Test mutation for nameconst: True, False, None.""" tree = Genome(nameconst_file).ast test_mutation = False testing_tree = deepcopy(tree) mutated_tree = MutateAST(target_idx=nameconst_expected_locs[0], mutation=test_mutation).visit( testing_tree ) mast = MutateAST(readonly=True) mast.visit(mutated_tree) # if statement is included with this file that will be picked up nc_locs = [loc for loc in mast.locs if loc.ast_class == "NameConstant"] assert len(nc_locs) == 4 for loc in nc_locs: # spot check on mutation from True to False if loc.lineno == 1 and loc.col_offset == 14: assert loc.op_type == test_mutation # spot check on not-mutated location still being None if loc.lineno == 7 and loc.col_offset == 22: assert loc.op_type is None
def test_create_mutation_and_run_trial(returncode, expected_status, monkeypatch, binop_file, binop_Add_LocIdx): """Mocked trial to ensure mutated cache files are removed after running.""" genome = Genome(source_file=binop_file) mutation_op = ast.Mult tag = sys.implementation.cache_tag expected_cfile = binop_file.parent / "__pycache__" / ".".join( [binop_file.stem, tag, "pyc"]) def mock_subprocess_run(*args, **kwargs): return CompletedProcess(args="pytest", returncode=returncode) monkeypatch.setattr(subprocess, "run", mock_subprocess_run) trial = run.create_mutation_run_trial( genome=genome, target_idx=binop_Add_LocIdx, mutation_op=mutation_op, test_cmds=["pytest"], max_runtime=10, ) # mutated cache files should be removed after trial run assert not expected_cfile.exists() assert trial.status == expected_status
def create_mutation_run_trial(genome: Genome, target_idx: LocIndex, mutation_op: Any, test_cmds: List[str]) -> MutantTrialResult: """Run a single mutation trial by creating a new mutated cache file, running the test commands, and then removing the mutated cache file. Args: genome: the genome to mutate target_idx: the mutation location mutation_op: the mutation operation test_cmds: the test commands to execute with the mutated code Returns: The mutation trial result """ LOGGER.debug("Running trial for %s", mutation_op) mutant = genome.mutate(target_idx, mutation_op, write_cache=True) mutant_trial = subprocess.run(test_cmds, capture_output=capture_output( LOGGER.getEffectiveLevel())) cache.remove_existing_cache_files(mutant.src_file) return MutantTrialResult(mutant=mutant, return_code=mutant_trial.returncode)
def add_five_to_mult_mutant(binop_file, stdoutIO): """Mutant that takes add_five op ADD to MULT. Fails if mutation code does not work.""" genome = Genome(source_file=binop_file) # this target is the add_five() function, changing add to mult target_idx = LocIndex(ast_class="BinOp", lineno=10, col_offset=11, op_type=ast.Add) mutation_op = ast.Mult mutant = genome.mutate(target_idx, mutation_op, write_cache=True) # uses the redirection for stdout to capture the value from the final output of binop_file with stdoutIO() as s: exec(mutant.mutant_code) assert int(s.getvalue()) == 25 return mutant
def test_MutateAST_visit_index_neg( i_order, lineno, col_offset, mut, index_file, index_expected_locs ): """Test mutation for Index: i[0], i[1], i[-1].""" tree = Genome(index_file).ast test_mutation = mut testing_tree = deepcopy(tree) mutated_tree = MutateAST(target_idx=index_expected_locs[i_order], mutation=test_mutation).visit( testing_tree ) mast = MutateAST(readonly=True) mast.visit(mutated_tree) assert len(mast.locs) == 4 for loc in mast.locs: # spot check on mutation from Index_NumNeg to Index_NumPos if loc.lineno == lineno and loc.col_offset == col_offset: assert loc.op_type == test_mutation # spot check on not-mutated location still being None if loc.lineno == 4 and loc.col_offset == 23: assert loc.op_type == "Index_NumPos"
def test_MutateAST_visit_read_only(binop_file): """Read only test to ensure locations are aggregated.""" tree = Genome(binop_file).ast mast = MutateAST(readonly=True) testing_tree = deepcopy(tree) mast.visit(testing_tree) # four locations from the binary operations in binop_file assert len(mast.locs) == 4 # tree should be unmodified assert ast.dump(tree) == ast.dump(testing_tree)
def test_MutateAST_visit_subscript(slice_file, slice_expected_locs): """Test Slice references within subscript.""" tree = Genome(slice_file).ast mast = MutateAST(readonly=True) mast.visit(tree) assert len(mast.locs) == len(slice_expected_locs) test_mutation = "Slice_UNegToZero" mutated_tree = MutateAST(target_idx=slice_expected_locs[2], mutation=test_mutation).visit(tree) mast.visit(mutated_tree) assert len(mast.locs) == len(slice_expected_locs) for loc in mast.locs: if loc.lineno == 5 and loc.col_offset == 15: assert loc.op_type == test_mutation # test one unmodified location if loc.lineno == 4 and loc.col_offset == 14: assert loc.op_type == "Slice_UnboundUpper"
def test_MutateAST_visit_boolop(boolop_file, boolop_expected_loc): """Test mutation of AND to OR in the boolop.""" tree = Genome(boolop_file).ast test_mutation = ast.Or # apply the mutation to the original tree copy testing_tree = deepcopy(tree) mutated_tree = MutateAST(target_idx=boolop_expected_loc, mutation=test_mutation).visit(testing_tree) # revisit in read-only mode to gather the locations of the new nodes mast = MutateAST(readonly=True) mast.visit(mutated_tree) # four locations from the binary operations in binop_file assert len(mast.locs) == 1 # there will only be one loc, but this still works # basedon the col and line offset in the fixture for compare_expected_loc for l in mast.locs: if l.lineno == 2 and l.col_offset == 11: assert l.op_type == test_mutation
def test_MutateAST_visit_augassign(augassign_file, augassign_expected_locs): """Test mutation for AugAssign: +=, -=, /=, *=.""" tree = Genome(augassign_file).ast test_mutation = "AugAssign_Div" testing_tree = deepcopy(tree) mutated_tree = MutateAST(target_idx=augassign_expected_locs[0], mutation=test_mutation).visit(testing_tree) mast = MutateAST(readonly=True) mast.visit(mutated_tree) assert len(mast.locs) == 4 for l in mast.locs: # spot check on mutation from Add tp Div if l.lineno == 1 and l.col_offset == 4: assert l.op_type == test_mutation # spot check on not-mutated location still being Mult if l.lineno == 5 and l.col_offset == 4: assert l.op_type == "AugAssign_Mult"
def create_mutation_run_trial(genome: Genome, target_idx: LocIndex, mutation_op: Any, test_cmds: List[str], max_runtime: float) -> MutantTrialResult: """Run a single mutation trial by creating a new mutated cache file, running the test commands, and then removing the mutated cache file. Args: genome: the genome to mutate target_idx: the mutation location mutation_op: the mutation operation test_cmds: the test commands to execute with the mutated code max_runtime: timeout for the trial Returns: The mutation trial result """ LOGGER.debug("Running trial for %s", mutation_op) mutant = genome.mutate(target_idx, mutation_op, write_cache=True) try: mutant_trial = subprocess.run( test_cmds, capture_output=capture_output(LOGGER.getEffectiveLevel()), timeout=max_runtime, ) return_code = mutant_trial.returncode except subprocess.TimeoutExpired: return_code = 3 cache.remove_existing_cache_files(mutant.src_file) return MutantTrialResult( mutant=MutantReport(src_file=mutant.src_file, src_idx=mutant.src_idx, mutation=mutant.mutation), return_code=return_code, )
def test_create_mutation_and_run_trial(returncode, expected_status, monkeypatch, binop_file): """Mocked trial to ensure mutated cache files are removed after running.""" genome = Genome(source_file=binop_file) # this target is the add_five() function, changing add to mult target_idx = LocIndex(ast_class="BinOp", lineno=10, col_offset=11, op_type=ast.Add) mutation_op = ast.Mult tag = sys.implementation.cache_tag expected_cfile = binop_file.parent / "__pycache__" / ".".join([binop_file.stem, tag, "pyc"]) def mock_subprocess_run(*args, **kwargs): return CompletedProcess(args="pytest", returncode=returncode) monkeypatch.setattr(subprocess, "run", mock_subprocess_run) trial = run.create_mutation_run_trial( genome=genome, target_idx=target_idx, mutation_op=mutation_op, test_cmds=["pytest"] ) # mutated cache files should be removed after trial run assert not expected_cfile.exists() assert trial.status == expected_status
def test_MutateAST_visit_compare(idx, mut_op, lineno, compare_file, compare_expected_locs): """Test mutation of the == to != in the compare op.""" tree = Genome(compare_file).ast # apply the mutation to the original tree copy testing_tree = deepcopy(tree) mutated_tree = MutateAST(target_idx=compare_expected_locs[idx], mutation=mut_op).visit( testing_tree ) # revisit in read-only mode to gather the locations of the new nodes mast = MutateAST(readonly=True) mast.visit(mutated_tree) assert len(mast.locs) == 3 # check that the lineno marked for mutation is changed, otherwise original ops should # still be present without modification for loc in mast.locs: if loc.lineno == lineno and loc.col_offset == 11: assert loc.op_type == mut_op else: assert loc.op_type in {ast.Eq, ast.Is, ast.In} # based on compare_file fixture
def test_covered_targets_source_file_TypeError(): """Targets with a NoneType source_file raises a TypeError.""" with pytest.raises(TypeError): genome = Genome() _ = genome.covered_targets
def test_covered_targets_coverage_file_TypeError(binop_file): """Targets with a NoneType coverage_file but valid source_file raises a TypeError.""" with pytest.raises(TypeError): genome = Genome(binop_file) genome.coverage_file = None _ = genome.covered_targets
def test_filter_codes_ValueError(): """Setting invalid filter codes on the Genome raises a ValueError.""" with pytest.raises(ValueError): genome = Genome() genome.filter_codes = ("asdf",)
def test_GenomeGroup_TypeError_source_file(): """GenomeGroup raises a TypeError adding a Genome without a set source_file.""" ggrp = GenomeGroup() with pytest.raises(TypeError): ggrp.add_genome(Genome())
def create_mutation_run_parallelcache_trial( genome: Genome, target_idx: LocIndex, mutation_op: Any, test_cmds: List[str], max_runtime: float) -> MutantTrialResult: """Similar to run.create_mutation_run_trial() but using the parallel cache directory settings. This function requires Python 3.8 and does not run with Python 3.7. Importantly, it has the identical signature to run.create_mutation_run_trial() and is substituted in the run.mutation_sample_dispatch(). Args: genome: the genome to mutate target_idx: the mutation location mutation_op: the mutation operation test_cmds: the test commands to execute with the mutated code max_runtime: timeout for the subprocess trial Returns: MutantTrialResult Raises: EnvironmentError: if Python version is less than 3.8 """ if sys.version_info < (3, 8): raise EnvironmentError( "Python 3.8 is required to use PYTHONPYCACHEPREFIX.") # Note in coverage reports this shows as untested code due to the subprocess dispatching # the 'slow' tests in `test_run.py` cover this. cache.check_cache_invalidation_mode() # create the mutant without writing the cache mutant = genome.mutate(target_idx, mutation_op, write_cache=False) # set up parallel cache structure parallel_cache = Path.cwd() / PARALLEL_PYCACHE_DIR / uuid.uuid4().hex resolved_source_parts = genome.source_file.resolve().parent.parts[ 1:] # type: ignore parallel_cfile = parallel_cache.joinpath( *resolved_source_parts) / mutant.cfile.name bytecode = importlib._bootstrap_external._code_to_timestamp_pyc( # type: ignore mutant.mutant_code, mutant.source_stats["mtime"], mutant.source_stats["size"]) LOGGER.debug("Writing parallel mutant cache file: %s", parallel_cfile) cache.create_cache_dirs(parallel_cfile) importlib._bootstrap_external._write_atomic( # type: ignore parallel_cfile, bytecode, mutant.mode, ) copy_env = os.environ.copy() copy_env["PYTHONPYCACHEPREFIX"] = str(parallel_cache) try: mutant_trial = subprocess.run( test_cmds, env=copy_env, capture_output=capture_output(LOGGER.getEffectiveLevel()), timeout=max_runtime + MULTI_PROC_TIMEOUT_BUFFER, ) return_code = mutant_trial.returncode except subprocess.TimeoutExpired: return_code = 3 LOGGER.debug("Removing parallel cache file: %s", parallel_cache.parts[-1]) shutil.rmtree(parallel_cache) return MutantTrialResult( mutant=MutantReport(src_file=mutant.src_file, src_idx=mutant.src_idx, mutation=mutant.mutation), return_code=return_code, )
def test_GenomeGroup_key_TypeError(key, binop_file): """Values that are not Path type keys raise a type error.""" with pytest.raises(TypeError): ggrp = GenomeGroup() ggrp[key] = Genome(binop_file)
def test_genome_ast(binop_file, binop_expected_locs): """Test that the AST builds expected targets.""" genome = Genome(source_file=binop_file) assert len(genome.targets) == 4 assert genome.targets == binop_expected_locs