def test_branch_coverage(self):
project_root = CLEAN_LINKED_LIST_ROOT
# trace_root = create_new_temp_dir()
trace_root = CLEAN_LINKED_LIST_ROOT
exclude_folders = ["venv", "dataset"]
cfg = ProjectCFG.create_from_path(project_root,
exclude_folders=exclude_folders,
use_cached_if_possible=False)
thorough.run_tests(CLEAN_LINKED_LIST_ROOT, trace_root, exclude_folders)
trace_reader = TraceReader(trace_root)
ll_py = str(CLEAN_LINKED_LIST_LL)
ll_py_cfg = cfg.module_cfgs[ll_py]
total_exercised = set()
available_branches = ll_py_cfg.branches
for node_id, path in zip(*trace_reader.get_traces_for(ll_py)):
df, size = read_as_dataframe(path)
covered = find_covered_branches(df, ll_py_cfg.branches)
total_exercised.update(covered)
print("Coverage")
print_percent("Branches covered", total_exercised, ll_py_cfg.branches)
print(available_branches)
print(total_exercised)
not_exercised_branches = set(available_branches) - set(total_exercised)
print("Not exercised branches total ({}): ".format(len(not_exercised_branches)), not_exercised_branches)
self.assertEqual(13, len(not_exercised_branches))
def test_astroid_util_definitions_uses(self):
project_cfg = ProjectCFG.create_from_path(
PROJECT_PATH, exclude_folders=exclude_folders)
line = 21
defs, uses = project_cfg.get_variables(ASTROID_UTIL, line)
self.assertIn("line", defs)
self.assertIn("function_def", uses)
def test_intra_method_pairs(self):
len_pairs = [
9, 8, 13, 19, 3, 20, 8, 5, 18, 16, 12, 19, 3, 18, 18, 24, 9, 2, 8
]
project_root = CLEAN_LINKED_LIST_ROOT
trace_root = create_new_temp_dir()
exclude_folders = ["venv"]
cfg = ProjectCFG.create_from_path(project_root,
exclude_folders=exclude_folders)
thorough.run_tests(CLEAN_LINKED_LIST_ROOT, trace_root, exclude_folders)
trace_reader = TraceReader(trace_root)
cppvi = VarIndexFactory.new_cpp_index(project_root, trace_root)
ll_py = str(CLEAN_LINKED_LIST_LL)
def get_pairs(trace_file_path):
np_array, _ = read_as_np_array(trace_file_path)
idx_pairs = analyze_trace_w_index(trace_file_path, cppvi)
def rename_vars(s):
return {(el[0], el[1]) for el in s}
idx_pairs = rename_vars(idx_pairs)
return idx_pairs
node_ids, paths = trace_reader.get_traces_for(ll_py)
for node_id, path, expected_pairs_count in zip(node_ids, paths,
len_pairs):
pairs = get_pairs(path)
self.assertEqual(
expected_pairs_count, len(pairs),
"Pairs count don't match for test case: {}".format(node_id))
def setUp(self) -> None:
relative_path = "../../dataset/linked_list_clean"
project_path = THIS_DIR / relative_path
self.project_cfg = ProjectCFG.create_from_path(
project_path.resolve(),
exclude_folders=["venv", "dataset"],
use_cached_if_possible=False)
self.ll_module_path = THIS_DIR / "../../dataset/linked_list_clean/core/ll.py"
self.ll_module_path = self.ll_module_path.resolve()
def __init__(self,
trace_root,
project_root,
exclude_folders=None,
max_trace_size=None):
self.project_cfg = ProjectCFG.create_from_path(
project_root,
exclude_folders=exclude_folders,
use_cached_if_possible=True)
self.max_trace_size = max_trace_size
self.trace_root = trace_root
self.trace_reader = TraceReader(trace_root)
self.vi = VarIndexFactory.new_py_index(project_root, trace_root)
self.cppvi = VarIndexFactory.new_cpp_index(project_root, trace_root)
def generic_experiment_coverage(project_root,
module_under_test_path,
scoring_functions,
coverage_boundaries_count=20,
max_trace_size=10,
coverage_metrics=None,
node_ids=None,
support=100):
cfg = ProjectCFG.create_from_path(project_root,
exclude_folders=COMMON_EXCLUDE)
if coverage_metrics is None:
coverage_metrics = DEFAULT_METRICS
coverage_boundaries = list(
zip(np.linspace(0.0, 0.95, num=coverage_boundaries_count),
np.linspace(0.05, 1.0, num=coverage_boundaries_count)))
generator = SuiteGenerator(project_root,
project_root,
COMMON_EXCLUDE,
max_trace_size=max_trace_size)
points = []
for metric in coverage_metrics:
for boundary in coverage_boundaries:
logger.debug(
"Coverage boundary: {boundary}, metric: {coverage_metric}",
boundary=boundary,
coverage_metric=metric)
suites = generator.fix_coverage_suites(
module_under_test_path=module_under_test_path,
coverage_metric=metric,
coverage_boundary=boundary,
n=support,
check_unique_items_covered=False,
test_cases=node_ids)
# if not suites:
# break
for suite in suites:
scores = [
scoring_function(suite)
for scoring_function in scoring_functions
]
point = (len(suite.test_cases), str(metric), *scores,
suite.coverage)
points.append(point)
return points
def setUpClass(cls):
relative_path = "../../dataset/dictionary"
cls.project_cfg = ProjectCFG.create_from_path(
os.path.join(THIS_DIR, relative_path),
exclude_folders=["venv", "dataset"])
multi_dict_class_name = "MultiDict"
multidict_cfg = None
# self.project.add_to_path()
for file_path, mod_cfg in cls.project_cfg.module_cfgs.items():
for cls_name in mod_cfg.class_cfgs:
if cls_name == multi_dict_class_name:
multidict_cfg = mod_cfg.class_cfgs[cls_name]
cls.multidict_cfg = multidict_cfg
def test_inter_method_pairs_test_interclass_pairs(self):
len_im_pairs = [
0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0
]
len_ic_pairs = [
5, 2, 5, 4, 1, 4, 2, 1, 5, 5, 7, 4, 1, 10, 7, 5, 9, 1, 4
]
project_root = CLEAN_LINKED_LIST_ROOT
trace_root = create_new_temp_dir()
exclude_folders = ["venv"]
cfg = ProjectCFG.create_from_path(project_root,
exclude_folders=exclude_folders)
thorough.run_tests(CLEAN_LINKED_LIST_ROOT, trace_root, exclude_folders)
trace_reader = TraceReader(trace_root)
cppvi = VarIndexFactory.new_py_index(project_root, trace_root)
ll_py = str(CLEAN_LINKED_LIST_LL)
def get_pairs(trace_file_path):
np_array, _ = read_as_np_array(trace_file_path)
scopes = read_scopes_for_trace_file(trace_file_path)
im_pairs, ic_pairs = analyze(trace_file_path, cppvi, scopes)
def rename_vars(s):
return {(el[0], el[1]) for el in s}
im_pairs = rename_vars(im_pairs)
ic_pairs = rename_vars(ic_pairs)
return im_pairs, ic_pairs
node_ids, paths = trace_reader.get_traces_for(ll_py)
for node_id, path, expected_im_len, expected_ic_len in zip(
node_ids, paths, len_im_pairs, len_ic_pairs):
im_pairs, ic_pairs = get_pairs(path)
self.assertEqual(
expected_im_len, len(im_pairs),
"Intermethod pairs count don't match for test case: {}".format(
node_id))
self.assertEqual(
expected_ic_len, len(ic_pairs),
"Intermethod pairs count don't match for test case: {}".format(
node_id))
def select_modules(project_root,
max_of_each_category=1,
min_pairs=0,
exclude_cfg=None):
if not exclude_cfg:
exclude_cfg = COMMON_EXCLUDE
cfg = ProjectCFG.create_from_path(project_root,
exclude_folders=exclude_cfg)
selected_modules = set()
stats = project_stats(cfg)
c = 0
for module_path, count_im, count_ic in sorted(stats,
key=operator.itemgetter(1),
reverse=True):
if count_im > 0:
if count_im < min_pairs:
continue
c += 1
if c > max_of_each_category:
break
else:
selected_modules.add(module_path)
c = 0
for module_path, count_im, count_ic in sorted(stats,
key=operator.itemgetter(2),
reverse=True):
if count_ic > 0:
if count_ic < min_pairs:
continue
c += 1
if c > max_of_each_category:
break
else:
selected_modules.add(module_path)
return selected_modules
module_under_test_absolute = os.path.join(
buggy_project.path, module_under_test_path)
covering_node_ids, paths = trace_reader.get_traces_for(
module_under_test_absolute)
covering_node_ids = set(covering_node_ids)
bad_node_ids = after - failing_on_assertion_node_ids
covering_node_ids -= bad_node_ids
not_failing_node_ids = covering_node_ids - failing_on_assertion_node_ids
not_failing_node_ids_as_params = " ".join(
not_failing_node_ids)
return_code = buggy_project.run_command(
f"pytest {not_failing_node_ids_as_params}")
if return_code == 0:
logger.warning("Test cases can be run individually")
cfg = ProjectCFG.create_from_path(buggy_project.path)
module_cfg = cfg.module_cfgs.get(
module_under_test_absolute)
if module_cfg:
mutants = get_mutants_of(
module_under_test_absolute)
stcov = StatementCoverage(
buggy_project.path,
buggy_project.path,
max_trace_size=max_trace_size)
brcov = BranchCoverage(
buggy_project.path,
buggy_project.path,
max_trace_size=max_trace_size)
ducov = DefUsePairsCoverage(
buggy_project.path,
def test_create_project_cfg(self):
project_cfg = ProjectCFG.create_from_path(
PROJECT_PATH, exclude_folders=exclude_folders)
self.assertIsNotNone(project_cfg)
self.assertIn(str(ASTROID_UTIL), project_cfg.module_cfgs.keys())
def generic_experiment_size(project_root,
module_under_test_path,
scoring_functions,
test_suite_sizes_count=30,
test_suite_sizes=None,
max_trace_size=10,
coverage_metrics=None,
node_ids=None,
support=100):
logger.debug("Running mutation experiment for {module}",
module=module_under_test_path)
if coverage_metrics is None:
coverage_metrics = DEFAULT_METRICS
cfg = ProjectCFG.create_from_path(project_root,
exclude_folders=COMMON_EXCLUDE)
if node_ids is None:
trace_reader = TraceReader(project_root)
node_ids, paths = trace_reader.get_traces_for(module_under_test_path)
total_cases = len(node_ids)
if test_suite_sizes is None:
if total_cases < test_suite_sizes_count:
test_suite_sizes_count = total_cases
test_suite_sizes = np.arange(test_suite_sizes_count) + 1
test_suite_sizes = [
int(c * total_cases / test_suite_sizes_count)
for c in test_suite_sizes
]
logger.debug("Testing test suites of sizes {ss}", ss=test_suite_sizes)
generator = SuiteGenerator(project_root,
project_root,
COMMON_EXCLUDE,
max_trace_size=max_trace_size)
points = []
no_suites_created_count_max = 3
for metric in coverage_metrics:
no_suites_created_count = 0
for sub_test_suites_size in test_suite_sizes:
logger.debug(
"Test suite size: {sub_test_suites_size}, metric: {metric}",
sub_test_suites_size=sub_test_suites_size,
metric=metric)
suites = generator.fix_sized_suites(
module_under_test_path=module_under_test_path,
coverage_metric=metric,
exact_size=sub_test_suites_size,
n=support,
check_unique_items_covered=False,
test_cases=node_ids)
if not suites:
no_suites_created_count += 1
else:
no_suites_created_count = 0
if no_suites_created_count > no_suites_created_count_max:
break
for suite in suites:
scores = [
scoring_function(suite)
for scoring_function in scoring_functions
]
point = (len(suite.test_cases), str(metric), *scores,
suite.coverage)
points.append(point)
return points