def try_evosuite(self,
classes_dir,
tests_src,
sut_class,
mutant,
evosuite_params=None):
junit = JUnit(java=self.java, classpath=classes_dir)
return (junit.run_with_mutant(self.suite_evosuite, sut_class, mutant)
if self.suite_evosuite else None)
def try_randoop(self,
classes_dir,
tests_src,
sut_class,
mutant,
randoop_params=None):
junit = JUnit(java=self.java, classpath=classes_dir)
return (junit.run_with_mutant(self.suite_randoop, sut_class, mutant)
if self.suite_randoop else None)
def test_run_junit(self):
junit = JUnit(self.java, self.classpath)
output = junit.exec(self.suite_dir, self.suite_classes_dir,
self.sut_class, self.suite_classes[0])
self.assertTrue(output.ok_tests_number > 0)
self.assertTrue(output.fail_tests == 0)
self.assertTrue(len(output.fail_test_set) == 0)
self.assertTrue(output.run_time > 0)
def test_run_junit_timeout(self):
junit = JUnit(self.java, self.classpath)
result = junit.exec(self.suite_dir, self.suite_classes_dir,
self.sut_class, self.suite_classes[0], 0)
self.assertEqual(0, result.ok_tests)
self.assertEqual(0, result.fail_tests)
self.assertEqual(0, len(result.fail_test_set))
self.assertEqual(0, result.run_time)
self.assertIsNone(result.coverage)
self.assertTrue(result.timeout)
def test_run_junit_with_mutant(self):
junit = JUnit(self.java, self.classpath)
mujava = MuJava(self.java, calculator_mutants_dir())
mutants = mujava.read_log()
mujava.compile_mutants(self.classpath, mutants)
output = junit.exec_with_mutant(self.suite_dir, self.suite_classes_dir,
self.sut_class, self.suite_classes[0],
mujava.read_log()[1])
self.assertTrue(output.ok_tests > 0)
self.assertTrue(output.fail_tests > 0)
self.assertTrue(len(output.fail_test_set) > 0)
self.assertTrue(output.run_time > 0)
def check_class_coverage(self, classes_dir, sut_class, mutant,
evo_test_result, ran_test_result):
evosuite_coverage = False
randoop_coverage = False
original_dir = os.path.join(mutant.dir[:mutant.dir.rfind(os.sep)],
'ORIGINAL')
original = Mutant(mid='ORIGINAL',
operator=None,
line_number=None,
method=None,
transformation=None,
dir=original_dir)
junit = JUnit(java=self.java, classpath=classes_dir)
orig_evosuite_result = (junit.run_with_mutant(self.suite_evosuite,
sut_class, original)
if self.suite_evosuite else None)
orig_ran_result = (junit.run_with_mutant(self.suite_randoop, sut_class,
original)
if self.suite_randoop else None)
if orig_evosuite_result == None and evo_test_result == None:
evosuite_coverage = True
elif orig_evosuite_result == None or evo_test_result == None:
evosuite_coverage = False
else:
evosuite_coverage = (orig_evosuite_result.coverage.class_coverage
== evo_test_result.coverage.class_coverage)
if orig_ran_result == None and ran_test_result == None:
randoop_coverage = True
elif orig_ran_result == None or ran_test_result == None:
randoop_coverage = False
else:
randoop_coverage = (orig_ran_result.coverage.class_coverage ==
ran_test_result.coverage.class_coverage)
return (evosuite_coverage and randoop_coverage)
class SuiteRunner():
def __init__(self, java, suite, suite_env):
self.java = java
self.suite = suite
self.suite_env = suite_env
self.junit = JUnit(java=self.java, classpath=suite_env.classes_dir)
def run_with_mutant(self, mutant):
return self.junit.run_with_mutant(self.suite, self.suite_env.sut_class,
mutant)
def get_tool_name(self):
return self.suite.tool_name
def try_evosuite_diff(self, classes_dir, sut_class, mutant_dir, project_dep):
junit = JUnit(java=project_dep.java, classpath=classes_dir)
res = junit.run_with_mutant(self.suite_evosuite_diff, sut_class, mutant_dir)
return res
def test_run_junit_timeout(self):
junit = JUnit(self.java, self.classpath)
with self.assertRaises(subprocess.TimeoutExpired):
junit.exec(self.suite_dir, self.suite_classes_dir,
self.sut_class, self.suite_classes[0], 0)
def try_randoop(self, classes_dir, sut_class, mutant_dir, project_dep):
junit = JUnit(java=project_dep.java, classpath=classes_dir)
return (junit.run_with_mutant(self.suite_randoop, sut_class,
mutant_dir)
if self.suite_randoop else None)
def run_test_suite(self, classes_dir, sut_class, mutant_dir, project_dep):
junit = JUnit(java=project_dep.java, classpath=classes_dir)
res = junit.run_with_mutant(self.test_suite, sut_class, mutant_dir)
return res
def run(self,
orig_project_dir,
mutants_dir,
sut_class,
randoop_params=None,
evosuite_diff_params=None,
evosuite_params=None,
nimrod_output_dir=None):
start_time = time()
nimrod_results = dict()
# Compile Project With Maven
print("Compiling the project with maven...")
_, orig_classes_dir = self.maven.compile(orig_project_dir, clean=True)
# Compile Tests with Maven
print("Compiling the tests with maven...")
_, orig_test_classes_dir = self.maven.test_compile(orig_project_dir)
# get outcome results
nimrod_output_dir = self._check_output_dir(
nimrod_output_dir if nimrod_output_dir else os.path.
join(orig_project_dir, OUTPUT_DIR))
self.junit = JUnit(java=self.java, classpath=orig_classes_dir)
# 1. Execute the dev tests (through Maven) - IMPORTANT: Skip this line and take the survived mutants (in case of Defects4J)
# **** Save the survived mutants
# 2. Take the survived mutants and execute TCE
tce_eqvs, tce_dups = self._try_tce(orig_project_dir, orig_classes_dir,
mutants_dir, sut_class)
# **** Remove TCE equivalents and duplicates (TODO: need log results before)
# self._del_mutants(mutants_dir, set(tce_eqvs+tce_dups))
# 3. Generate automatic tests with EvoSuite and Randoop (based on original)
auto_generated_tests_dir = self._gen_automatic_tests(
orig_classes_dir, mutants_dir, nimrod_output_dir, sut_class,
randoop_params, evosuite_params)
# Save the automatic tests alongside the dev tests (combined test set)
self._combine_tests(orig_test_classes_dir, auto_generated_tests_dir)
# Exec tests on original program
print("Executing tests on original...")
num_tests, num_failures, num_errors, num_skipped, failed_tests = self.maven.test(
orig_project_dir, sut_class)
# Stop execution in case of flaky tests
if (num_failures > 0 or num_errors > 0):
print(
"*** ERROR - Flaky tests or error in automatic test generation. Please check executing: mvn test."
)
print(
"num_tests: {0}, num_failures: {1}, num_errors: {2}, num_skipped: {3}, failed_tests: {4}"
.format(num_tests, num_failures, num_errors, num_skipped,
failed_tests))
return
#Configure mutant tool, compile mutants dir, and return the mutants set
nimrod_non_equivs = set()
nimrod_survived_muts = set()
mutants = self.get_mutants(orig_classes_dir, mutants_dir)
# print("Total mutants: {0}".format(len(mutants))) # esse numero nao eh real, pois soh pega os mutantes do arq de log
print("Mutant analysis phase started")
for mutant in mutants:
# Does not exec TCE eqvs and dupl mutants
if (mutant.mid[mutant.mid.index('_') + 1:] in tce_eqvs):
if ('$' not in mutant.method
): # ib case of $ means interal class which is a TCE error
print("Mutant {0} is equivalent".format(mutant.mid))
nimrod_results[mutant.mid] = NimrodResult(
mutant.mid, mutant.operator, mutant.line_number,
mutant.method, mutant.transformation, True, '', '', '',
'', '', '', '')
continue
if (os.path.isdir(mutant.dir)):
evosuite_diff_output_dir = self._gen_evosuite_diff(
sut_class, evosuite_diff_params, orig_classes_dir,
auto_generated_tests_dir, mutant)
# test_result = self._exec_junit(evosuite_diff_output_dir, sut_class, mutant)
test_result = self.try_evosuite_diff(orig_classes_dir,
evosuite_diff_output_dir,
sut_class, mutant,
evosuite_diff_params)
if test_result.fail_tests > 0 or test_result.timeout: #If killed by EvoSuiteR Diff
print("Mutant {0} is NOT equivalent. Killed by {1}".format(
mutant.mid,
self._get_test_files(
evosuite_diff_output_dir.suite_dir)))
nimrod_non_equivs.add(mutant)
nimrod_results[mutant.mid] = NimrodResult(
mutant.mid, mutant.operator, mutant.line_number,
mutant.method, mutant.transformation, False, True,
test_result.fail_tests,
self._get_test_files(
evosuite_diff_output_dir.suite_dir),
test_result.timeout, 1, False, '')
else:
# print("Mutant {0} was NOT killed by EvoSuiteR".format(mutant.mid))
# self._remove_wrong_test(evosuite_diff_output_dir.suite_dir)
nimrod_survived_muts.add(mutant)
# Junta todos os testes gerados com os testes do desenvolvedor
self._combine_tests(orig_test_classes_dir, auto_generated_tests_dir)
# Executa no programa original
num_tests, num_failures, num_errors, num_skipped, failed_tests = self.maven.test(
orig_project_dir, sut_class)
self._copy_coverage_report_to_output_dir(orig_project_dir,
nimrod_output_dir, 'original')
orig_coverage = self._get_coverage(orig_project_dir, sut_class)
# Stop execution in case of flaky tests
if (num_failures > 0 or num_errors > 0):
print(
"*** ERROR - Flaky tests or error in automatic test generation. Please check executing: mvn test."
)
print(
"num_tests: {0}, num_failures: {1}, num_errors: {2}, num_skipped: {3}, failed_tests: {4}"
.format(num_tests, num_failures, num_errors, num_skipped,
failed_tests))
return
# Próximo passo calcular o Coverage e montar o Rank
for mutant in nimrod_survived_muts:
# Copia o mutante para a pasta correta em target/classes
copy_dir(mutant.dir, orig_classes_dir)
try:
# Executa o os testes combinados com o mutante que nao morreu com EvoSuiteR - Pega Coverage
num_tests, num_failures, num_errors, num_skipped, failed_tests = self.maven.test(
orig_project_dir, sut_class)
self._copy_coverage_report_to_output_dir(
orig_project_dir, nimrod_output_dir, mutant.mid)
mut_coverage = self._get_coverage(orig_project_dir, sut_class)
executions, test_cases_cps = self._get_mutation_line_info(
mut_coverage, mutant.line_number)
diff_lines = self._get_diff_coverage_lines(
orig_coverage, mut_coverage)
if (num_failures > 0 or num_errors > 0):
print(
"Mutant {0} is NOT equivalent. Killed by {1}.".format(
mutant.mid, failed_tests))
nimrod_results[mutant.mid] = NimrodResult(
mutant.mid, mutant.operator, mutant.line_number,
mutant.method, mutant.transformation, False, True,
num_failures + num_errors, failed_tests, False,
executions, False if len(diff_lines) > 0 else True,
len(diff_lines))
else:
print("Mutant {0} may be equivalent. Coverage is {1}.".
format(
mutant.mid,
'Different' if len(diff_lines) > 0 else 'Equal'))
nimrod_results[mutant.mid] = NimrodResult(
mutant.mid, mutant.operator, mutant.line_number,
mutant.method, mutant.transformation, False, False, '',
'', False,
executions, False if len(diff_lines) > 0 else True,
len(diff_lines))
except Exception as e:
print("**ERROR: Test timedout to mutant {0}".format(
mutant.mid))
# Adiciona mutante como diferenete e justifica com Timeout
print("Mutant {0} is NOT equivalent. Timeout error.".format(
mutant.mid))
nimrod_results[mutant.mid] = NimrodResult(
mutant.mid, mutant.operator, mutant.line_number,
mutant.method, mutant.transformation, False, True, '', '',
True, 0, False, '')
self._print_output(nimrod_results)
self.write_to_csv(nimrod_results,
output_dir=nimrod_output_dir,
filename='nimrod.csv',
exclude_if_exists=True)
print('Finished Nimrod analysis of {0} mutants in {1} secs: '.format(
len(nimrod_results), round((time() - start_time), 2)))
def __init__(self, java, suite, suite_env):
self.java = java
self.suite = suite
self.suite_env = suite_env
self.junit = JUnit(java=self.java, classpath=suite_env.classes_dir)
