def get_num_passing_failing_variants(mutated_project_dir, list_of_stms,
spectrum_coverage_prefix):
failing_passing_variants_of_stms = init_num_of_failing_passing_variants(
list_of_stms)
variants_list = get_all_variant_dirs(mutated_project_dir)
total_fails = 0
total_passes = 0
for variant_dir in variants_list:
test_coverage_dir = get_test_coverage_dir(variant_dir)
spectrum_failed_file = get_spectrum_failed_coverage_file_name_with_version(
spectrum_coverage_prefix)
spectrum_failed_coverage_file_dir = join_path(test_coverage_dir,
spectrum_failed_file)
spectrum_passed_file = get_spectrum_passed_coverage_file_name_with_version(
spectrum_coverage_prefix)
spectrum_passed_coverage_file_dir = join_path(test_coverage_dir,
spectrum_passed_file)
if os.path.isfile(spectrum_failed_coverage_file_dir):
failing_passing_variants_of_stms = read_data_from_coverage_file(
failing_passing_variants_of_stms,
spectrum_failed_coverage_file_dir, VARIANT_NUM_OF_FAILS)
total_fails += 1
if not os.path.isfile(
spectrum_failed_coverage_file_dir) and os.path.isfile(
spectrum_passed_coverage_file_dir):
failing_passing_variants_of_stms = read_data_from_coverage_file(
failing_passing_variants_of_stms,
spectrum_passed_coverage_file_dir, VARIANT_NUM_OF_PASSES)
total_passes += 1
return failing_passing_variants_of_stms, total_fails, total_passes
def read_coverage_file(mutated_project_dir):
variants_dir = get_variants_dir(mutated_project_dir)
variants_list = os.listdir(variants_dir)
data = {}
for variant in variants_list:
variant_dir = get_variant_dir(mutated_project_dir, variant)
test_coverage_dir = get_test_coverage_dir(variant_dir)
coverage_file = join_path(test_coverage_dir, SPECTRUM_FAILED_COVERAGE_FILE_NAME)
if os.path.isfile(coverage_file):
data[variant] = []
try:
tree = ET.parse(coverage_file)
root = tree.getroot()
project = root.find("project")
for package in project:
for file in package:
for line in file:
id = line.get('featureClass') + "." + line.get('featureLineNum')
if id not in data[variant] and int(line.get('count')) != 0:
data[variant].append(id)
except:
logging.info("Exception when parsing %s", coverage_file)
return data
def statement_coverage(variant_dir, spectrum_coverage_prefix):
global NEW_SPECTRUM_PASSED_COVERAGE_FILE_NAME
NEW_SPECTRUM_PASSED_COVERAGE_FILE_NAME = spectrum_coverage_prefix + SPECTRUM_PASSED_COVERAGE_FILE_NAME
test_coverage_dir = get_test_coverage_dir(variant_dir)
spectrum_passed_coverage_file = join_path(test_coverage_dir, NEW_SPECTRUM_PASSED_COVERAGE_FILE_NAME)
if not os.path.isfile(spectrum_passed_coverage_file):
spectrum_passed_coverage_file = join_path(test_coverage_dir, SPECTRUM_PASSED_COVERAGE_FILE_NAME)
num_of_stm = 0
untested_stm = 0
if os.path.isfile(spectrum_passed_coverage_file):
try:
tree = ET.parse(spectrum_passed_coverage_file)
root = tree.getroot()
project = root.find("project")
for package in project:
for file in package:
for line in file:
num_of_stm += 1
if (int(line.get("count")) == 0):
untested_stm += 1
except:
logging.info("Exception when parsing %s", spectrum_passed_coverage_file)
else:
logging.info("spectrum passed coveraged file does not exist in %s", variant_dir)
coverage_rate = (num_of_stm - untested_stm) / num_of_stm
return coverage_rate
def check_coverage_file_quantity(variant_dir):
test_case_names = get_all_test_case_name_from_source_files(variant_dir)
coverage_dir = get_test_coverage_dir(variant_dir)
coverage_file_names = set([
get_file_name(e).replace(f".{TEST_COVERAGE_FILE_EXTENSION}", "")
for e in get_all_coverage_file_paths_in_dir(coverage_dir)
])
assert test_case_names == coverage_file_names, f"\n---\nTest case names: {test_case_names}\nCoverage file names: {coverage_file_names}\n\nVariant dir: {variant_dir}\n---"
def run_junit_test_cases_with_coverage(variant_dir,
lib_paths=None,
halt_on_failure=False,
halt_on_error=True,
custom_ant=None):
if lib_paths is None:
lib_paths = []
logger.info(
f"Running JUnit Test for variant [{get_file_name_without_ext(variant_dir)}] - Using custom ant [{custom_ant}]"
)
src_dir = get_src_dir(variant_dir)
test_dir = get_test_dir(variant_dir)
src_classes_dir = get_compiled_source_classes_dir(variant_dir)
test_classes_dir = get_compiled_test_classes_dir(variant_dir)
test_coverage_dir = get_test_coverage_dir(variant_dir)
output_log = execute_shell_command(
f'/bin/sh {JUNIT_PLUGIN_PATH}',
extra_args=[
{
"-src": src_dir
},
{
"-test": test_dir
},
{
"-build.classes": src_classes_dir
},
{
"-build.testclasses": test_classes_dir
},
{
"-report.coveragedir": test_coverage_dir
},
{
"-junit.haltonfailure": "yes" if halt_on_failure else "no"
},
{
"-ant.name": custom_ant
},
{
"-lib_path": ":".join(lib_paths)
},
],
log_to_file=True)
is_test_failure = re.search(
"(Failures: [1-9]+|Errors: [1-9]+|BUILD FAILED)", output_log)
if is_test_failure:
if halt_on_failure or (halt_on_error
and "BUILD FAILED" in is_test_failure.group()):
logger.fatal(
"Some test cases were failed, see log for more detail\n{}".
format(output_log))
raise RuntimeError("Test case failures")
return False
return True
def check_buggy_statements_executed_in_failed_tests(variant_dir,
feature_buggy_statements):
test_coverage_dir = get_test_coverage_dir(variant_dir)
failed_coverage_dir = get_failed_test_coverage_dir(variant_dir)
if failed_coverage_dir is None:
return
buggy_statement_dict = get_buggy_statements_in_variant_source_code(
test_coverage_dir, feature_buggy_statements)
for failed_coverage_file_path in get_all_coverage_file_paths_in_dir(
failed_coverage_dir):
validate_buggy_statement_count_in_coverage_file(
failed_coverage_file_path, buggy_statement_dict)
def check_variant_final_test_output(variant_dir, junit_mode=JunitMode.FAST):
coverage_dir = get_test_coverage_dir(variant_dir)
if is_path_exist(join_path(variant_dir, FAILED_TEST_FLAG_FILE_NAME)):
if junit_mode == JunitMode.FULL_COVERAGE and not is_path_exist(
join_path(coverage_dir, FAILED_TEST_COVERAGE_FOLDER_NAME)):
return None
return False
elif is_path_exist(join_path(variant_dir, PASSED_TEST_FLAG_FILE_NAME)):
if junit_mode == JunitMode.FULL_COVERAGE and not is_path_exist(
join_path(coverage_dir, PASSED_TEST_COVERAGE_FOLDER_NAME)):
return None
return True
else:
return None
def check_spectrum_coverage_file_aggregated_from_all_tests(variant_dir):
test_coverage_dir = get_test_coverage_dir(variant_dir)
passed_coverage_dir = get_passed_test_coverage_dir(variant_dir)
if passed_coverage_dir is None:
return
passed_spectrum_coverage_file = get_passed_spectrum_coverage_file_path_with_version(
test_coverage_dir)
validate_spectrum_coverage_aggregated_completely(
passed_spectrum_coverage_file, passed_coverage_dir)
failed_coverage_dir = get_failed_test_coverage_dir(variant_dir)
if failed_coverage_dir is None:
return
failed_spectrum_coverage_file = get_failed_spectrum_coverage_file_path_with_version(
test_coverage_dir)
validate_spectrum_coverage_aggregated_completely(
failed_spectrum_coverage_file, failed_coverage_dir)
def rebuild_incremental_spectrum_coverages_for_variant(variant_dir,
version=""):
raw_version = version
test_coverage_dir = get_test_coverage_dir(variant_dir)
failed_coverage_dir = join_path(test_coverage_dir,
FAILED_TEST_COVERAGE_FOLDER_NAME)
failed_coverage_file_paths = TestingCoverageManager.get_all_coverage_file_paths(
failed_coverage_dir)
passed_coverage_dir = join_path(test_coverage_dir,
PASSED_TEST_COVERAGE_FOLDER_NAME)
passed_coverage_file_paths = TestingCoverageManager.get_all_coverage_file_paths(
passed_coverage_dir)
test_suites = get_incremental_test_suites(failed_coverage_file_paths,
passed_coverage_file_paths)
for i, test_suite in enumerate(test_suites):
coverage_file_paths = test_suite
version = f"INoT_{i + 1}_{raw_version}"
build_incremental_spectrum_coverages_with_each_test_suite(
coverage_file_paths, test_coverage_dir, version)
def get_coverage_infor_of_variants(variant, variant_dir, failing_variants,
features_coverage_info, search_spaces,
spectrum_coverage_prefix):
test_coverage_dir = get_test_coverage_dir(variant_dir)
spectrum_failed_file = get_spectrum_failed_coverage_file_name_with_version(
spectrum_coverage_prefix)
spectrum_failed_coverage_file_dir = join_path(test_coverage_dir,
spectrum_failed_file)
spectrum_passed_file = get_spectrum_passed_coverage_file_name_with_version(
spectrum_coverage_prefix)
spectrum_passed_coverage_file_dir = join_path(test_coverage_dir,
spectrum_passed_file)
if variant in failing_variants:
if os.path.isfile(spectrum_failed_coverage_file_dir):
features_coverage_info = read_coverage_info(
variant, features_coverage_info, search_spaces,
spectrum_failed_coverage_file_dir, VARIANTS_FAILED)
if os.path.isfile(spectrum_passed_coverage_file_dir):
features_coverage_info = read_coverage_info(
variant, features_coverage_info, search_spaces,
spectrum_passed_coverage_file_dir, VARIANTS_FAILED)
else:
if os.path.isfile(spectrum_failed_coverage_file_dir):
features_coverage_info = read_coverage_info(
variant, features_coverage_info, search_spaces,
spectrum_failed_coverage_file_dir, VARIANTS_PASSED)
if os.path.isfile(spectrum_passed_coverage_file_dir):
features_coverage_info = read_coverage_info(
variant, features_coverage_info, search_spaces,
spectrum_passed_coverage_file_dir, VARIANTS_PASSED)
return features_coverage_info
def rebuild_spectrum_coverage_for_variant(variant_dir,
version=None,
random=True):
test_coverage_dir = get_test_coverage_dir(variant_dir)
failed_coverage_dir = join_path(test_coverage_dir,
FAILED_TEST_COVERAGE_FOLDER_NAME)
if is_path_exist(failed_coverage_dir):
spectrum_failed_coverage_file_path = get_failed_spectrum_coverage_file_path_with_version(
test_coverage_dir, version)
rebuild_spectrum_coverage(
input_coverage_dir=failed_coverage_dir,
spectrum_output_path=spectrum_failed_coverage_file_path,
random=random)
passed_coverage_dir = join_path(test_coverage_dir,
PASSED_TEST_COVERAGE_FOLDER_NAME)
if is_path_exist(passed_coverage_dir):
spectrum_passed_coverage_file_path = get_passed_spectrum_coverage_file_path_with_version(
test_coverage_dir, version)
rebuild_spectrum_coverage(
input_coverage_dir=passed_coverage_dir,
spectrum_output_path=spectrum_passed_coverage_file_path,
random=random)
def is_var_bug(mutated_project_dir,
filter_coverage,
spectrum_coverage_prefix=""):
num_of_failing_variants = 0
num_of_passing_variants = 0
variants_dir = get_variants_dir(mutated_project_dir)
variants_list = list_dir(variants_dir)
for variant in variants_list:
variant_dir = get_variant_dir(mutated_project_dir, variant)
test_coverage_dir = get_test_coverage_dir(variant_dir)
stm_coverage = statement_coverage(variant_dir,
spectrum_coverage_prefix)
spectrum_failed_file = get_spectrum_failed_coverage_file_name_with_version(
spectrum_coverage_prefix)
failed_file = join_path(test_coverage_dir, spectrum_failed_file)
if os.path.isfile(failed_file):
num_of_failing_variants += 1
elif stm_coverage >= filter_coverage:
num_of_passing_variants += 1
if num_of_failing_variants >= 1 and num_of_passing_variants >= 1:
return 1
return 0
def rebuild_spectrum_coverage_with_target_coverage(mutated_project_dir,
target_coverage):
"""
require all variants to have the same coverage level
interrupt immediately if no solution found for a specific variant
"""
mutated_variant_dirs = get_all_variant_dirs(mutated_project_dir, sort=True)
test_coverage_container = []
version = f"{int(target_coverage * 100)}"
# find optimal test cases
for mutated_variant_dir in mutated_variant_dirs[:]:
if True or "model_m_ca4_0004" in mutated_variant_dir:
print("-" * 20)
print(mutated_variant_dir)
test_coverage_dir = get_test_coverage_dir(mutated_variant_dir)
failed_test_coverage_dir = get_failed_test_coverage_dir(
mutated_variant_dir)
passed_test_coverage_dir = get_passed_test_coverage_dir(
mutated_variant_dir)
print(
TestingCoverageManager.print_coverage_summary(
failed_test_coverage_dir, passed_test_coverage_dir))
