def initials_to_DS(self):
ds = software.sfl_diagnoser.Diagnoser.dynamicSpectrum.dynamicSpectrum()
ds.TestsComponents = copy.deepcopy(
[Experiment_Data().POOL[test] for test in self.initial_tests])
ds.probabilities = list(Experiment_Data().PRIORS)
ds.error = [self.error[test] for test in self.initial_tests]
ds.tests_names = list(self.initial_tests)
return ds
def calc_wasted_components(self):
components = map(lambda x: x[0], self.get_components_probabilities())
wasted = 0.0
for b in Experiment_Data().BUGS:
if b not in components:
return float('inf')
wasted += components.index(b)
return wasted / len(Experiment_Data().BUGS)
def get_components_vectors(self):
components = {}
for test in self.initial_tests:
for component in Experiment_Data().POOL[test]:
components.setdefault(
Experiment_Data().COMPONENTS_NAMES[component],
[]).append(test)
return components
def save_ds_to_matrix_file(ds, out_file):
tests_details = map(
lambda details: (str(details[0]),
map(lambda c: Experiment_Data().COMPONENTS_NAMES[c],
details[1]), details[2]),
list(zip(ds.tests_names, ds.TestsComponents, ds.error)))
write_planning_file(
out_file,
map(lambda c: Experiment_Data().COMPONENTS_NAMES[c],
Experiment_Data().BUGS), tests_details)
def __init__(self, initial_tests, error):
self.TestsComponents = copy.deepcopy(
[Experiment_Data().POOL[test] for test in initial_tests])
self.prior_probs = list(Experiment_Data().PRIORS)
self.M_matrix = list(
map(
lambda test: list(
map(lambda comp: 1
if comp in test else 0, range(len(self.prior_probs)))),
self.TestsComponents))
self.initial_e_vector = [error[test] for test in initial_tests]
def new_comps(ei):
ei.diagnose()
probabilities = []
optionals = ei.get_optionals_actions()
components = set()
for test in ei.initial_tests:
trace = Experiment_Data().POOL[test]
components += set(trace)
for test in optionals:
trace = Experiment_Data().POOL[test]
probabilities.append(len([c for c in trace if c not in components]))
return [x / sum(probabilities) for x in probabilities]
def pipeline(folder, output):
matrices = [
file for file in os.listdir(folder) if file.endswith('.matrix')
]
matrices = sorted(matrices,
key=file2tuple) # for logic traversal of matrices.
print(f'total of {len(matrices)} matrices')
matrices = list(filter(filter_low, matrices))
# from_matrix = '14_14_11.matrix'
# from_index = dict(map(reversed,enumerate(matrices)))[from_matrix]
# to_matrix = '14_14_20.matrix'
# to_index = dict(map(reversed,enumerate(matrices)))[to_matrix]
# matrices = matrices[from_index:to_index + 1]
print(f'remaining {len(matrices)} matrices')
# print(matrices[0])
# return
with open(output, 'w', newline='') as f:
writer = csv.writer(f)
writer.writerow(
('matrix_name', 'sample', '#components', '#tests',
'#failing_tests', 'error_vector', 'o2d_time', 'o2d_mhs_time',
'o2d_best_cardinality', 'o2d_mhs_best_cardinality',
'o2d_mean_cardinality', 'o2d_mhs_mean_cardinality', 'o2d_output',
'o2d_mhs_output'))
f.flush()
for matrix_file in matrices:
print(f'diagnosing matrix {matrix_file}')
diagnoser, smart_mhs_diagnoser, error, initials, _ = readPlanningFile(
os.path.join(folder, matrix_file))
error_vec = list(
map(operator.itemgetter(1),
sorted(error.items(), key=operator.itemgetter(0))))
num_of_comps = len(Experiment_Data().COMPONENTS_NAMES.keys())
all_tests = Experiment_Data().POOL.keys()
num_of_tests = len(all_tests)
num_failing_tests = error_vec.count(1)
sample_num = int(matrix_file.split('.')[0].split('_')[2])
# if sample_num != 2:
# continue
print(
f'comps: {num_of_comps}, tests: {num_of_tests}, failing: {num_failing_tests}'
)
for results, time, best_diag_card, mean_card in compare_with_smart_mhs(
diagnoser, error, all_tests, smart_mhs_diagnoser):
writer.writerow((matrix_file, sample_num, num_of_comps,
num_of_tests, num_failing_tests, error_vec,
*time, *best_diag_card, *mean_card, *results))
f.flush()
print()
print('\n')
def components_activity(ei, aggregation):
ei.diagnose()
probabilities = []
optionals = ei.get_optionals_actions()
activity = {}
for test in ei.initial_tests:
trace = Experiment_Data().POOL[test]
for c in trace:
activity[c] = activity.get(c, 0) + 1
for test in optionals:
trace = Experiment_Data().POOL[test]
probabilities.append(aggregation([activity.get(c, 0) for c in trace]))
return [x / sum(probabilities) for x in probabilities]
def fail_sum(ei):
ei.diagnose()
probabilities = []
optionals = ei.get_optionals_actions()
failed_components = {}
for test in ei.initial_tests:
if ei.error[test] == 0:
continue
trace = Experiment_Data().POOL[test]
for c in trace:
failed_components[c] = failed_components.get(c, 0) + 1
for test in optionals:
trace = Experiment_Data().POOL[test]
probabilities.append(sum([failed_components.get(c, 0) for c in trace]))
return [x / sum(probabilities) for x in probabilities]
def fail_count(ei):
ei.diagnose()
probabilities = []
optionals = ei.get_optionals_actions()
failed_components = set()
for test in ei.initial_tests:
if ei.error[test] == 0:
continue
trace = Experiment_Data().POOL[test]
for c in trace:
failed_components.add(c)
for test in optionals:
trace = Experiment_Data().POOL[test]
probabilities.append(sum([c for c in trace if c in failed_components]))
return [x / sum(probabilities) for x in probabilities]
def diagnose_all_combinations(inst, error, faulty_comp_prob,
faulty_output_prob, uncertain_tests):
components_dict = Experiment_Data().COMPONENTS_NAMES
diagnoses = {}
# counter = 0
for obs, obs_prob in diagnoser_utils.uncertain_observation_iterator(
error, faulty_output_prob, uncertain_tests):
# print(''.join(map(str, obs.values())) + str(obs_prob))
diagnoser = inst(obs)
if all(err == 0 for err in obs.values()):
diagnoses[tuple()] = obs_prob
continue
diagnoser.diagnose(normalize=False)
for diag in diagnoser.diagnoses:
comps = tuple(sorted([components_dict[c] for c in diag.diagnosis]))
# if comps == ('c10', 'c9'):
# counter += 1
# comps = tuple(sorted(diag.diagnosis))
old_diag_prob = diagnoses.get(comps, 0)
diagnoses[comps] = old_diag_prob + diag.probability * obs_prob
diagnoses_with_probs = diagnoses.items()
sum_probs = sum(prob for _, prob in diagnoses_with_probs)
# print(counter)
return [(diag, prob / sum_probs) for diag, prob in diagnoses_with_probs]
def num_components(ei):
optionals = ei.get_optionals_actions()
probabilities = []
for test in optionals:
trace = Experiment_Data().POOL[test]
probabilities.append(len(trace))
return [x / sum(probabilities) for x in probabilities]
def components_by_similarity(ei, aggregation, similarity):
ei.diagnose()
probabilities = []
optionals = ei.get_optionals_actions()
e = [ind for ind, _ in enumerate(ei.error)]
components = {}
for test in ei.initial_tests:
trace = Experiment_Data().POOL[test]
for c in trace:
components[c] = components.get(c, []) + [test]
similarities = dict([(c, similarity(components[c], e))
for c in components])
for test in optionals:
trace = Experiment_Data().POOL[test]
probabilities.append(
aggregation([similarities.get(c, 0) for c in trace]))
return [x / sum(probabilities) for x in probabilities]
def compute_pass_prob(self, action):
trace = Experiment_Data().POOL[action]
probs = dict(self.get_components_probabilities())
pass_Probability = 1.0
for comp in trace:
pass_Probability *= 0.999 # probability of 1 fault for each 1000 lines of code
if comp in probs:
pass_Probability *= (1 - probs[comp]) # add known faults
return round(pass_Probability, 6)
def components_probabilities(ei, aggregation):
components_probs = dict(ei.get_components_probabilities())
optionals = ei.get_optionals_actions()
probabilities = []
for test in optionals:
trace = Experiment_Data().POOL[test]
probabilities.append(
reduce(aggregation, [components_probs.get(c, 0) for c in trace]))
return [x / sum(probabilities) for x in probabilities]
def count_different_cases(self):
"""
:return: the number of different test cases in the diagnosis
"""
optional_tests = list(
map(
lambda enum: enum[1],
filter(lambda enum: enum[0] in self.initial_tests,
enumerate(Experiment_Data().POOL))))
return len(set(map(str, optional_tests)))
def get_named_diagnoses(self):
self.diagnose()
named_diagnoses = []
for diagnosis in self.diagnoses:
named = Diagnosis.Diagnosis(
map(lambda id: Experiment_Data().COMPONENTS_NAMES[id],
diagnosis.diagnosis))
named.probability = diagnosis.probability
named_diagnoses.append(named)
return named_diagnoses
def split_components_probabilities(ei):
components_probs = dict(ei.get_components_probabilities())
half_components_probabilities = sum(components_probs.values()) / 2.0
optionals = ei.get_optionals_actions()
probabilities = []
for test in optionals:
trace = Experiment_Data().POOL[test]
probabilities.append(1 - abs(half_components_probabilities - reduce(
lambda x, y: x + y, [components_probs.get(c, 0) for c in trace])))
return list(zip(optionals,
[x / sum(probabilities) for x in probabilities]))
def calc_precision_recall(self):
self.diagnose()
recall_accum = 0
precision_accum = 0
validComps = [
x for x in range(
max(reduce(list.__add__,
Experiment_Data().POOL.values())))
if x not in Experiment_Data().BUGS
]
for d in self.diagnoses:
dg = d.diagnosis
pr = d.probability
precision, recall = ExperimentInstance.precision_recall_diag(
Experiment_Data().BUGS, dg, pr, validComps)
if (recall != "undef"):
recall_accum = recall_accum + recall
if (precision != "undef"):
precision_accum = precision_accum + precision
return precision_accum, recall_accum
def __init__(self, diagnoses, initial_tests, error, pool=None, bugs=None):
self.diagnoses = diagnoses
self.initial_tests = initial_tests
self.error = error
self.pool = pool
if pool == None:
self.pool = Experiment_Data().POOL
self.bugs = bugs
if bugs == None:
self.bugs = Experiment_Data().BUGS
self.components = set(
reduce(
list.__add__,
map(
lambda test: test[1],
filter(lambda test: test[0] in self.initial_tests,
self.pool.items())), []))
self.metrics = self._calculate_metrics()
for key, value in self.metrics.items():
setattr(self, key, value)
def readPlanningFile(fileName, delimiter=";", cut=False, use_smart_mhs=False):
lines = open(fileName, "r").readlines()
lines = [x.replace("\n", "") for x in lines]
sections = [
"[Description]", "[Components names]", "[Priors]", "[Bugs]",
"[InitialTests]", "[TestDetails]"
]
sections = [lines.index(x) for x in sections]
description, components_names, priorsStr, BugsStr, InitialsStr, TestDetailsStr = tuple(
[
lines[x[0] + 1:x[1]]
for x in zip(sections, sections[1:] + [len(lines)])
])
priors = eval(priorsStr[0])
bugs = eval(BugsStr[0])
initials = eval(InitialsStr[0])
try:
components = dict(
map(lambda x: x if isinstance(x, tuple) else eval(x),
eval(components_names[0].replace(delimiter, ',') + ',')))
except:
components = dict(
eval(eval(components_names[0].replace(delimiter, ','))))
testsPool = {}
estimatedTestsPool = {}
error = {}
for td in TestDetailsStr:
tup = tuple(td.split(delimiter))
ind, actualTrace, err = None, None, None
if len(tup) == 3:
ind, actualTrace, err = tup
if len(tup) == 4:
ind, actualTrace, estimatedTrace, err = tup
estimatedTestsPool[ind] = eval(estimatedTrace)
actualTrace = eval(actualTrace)
err = int(err)
testsPool[ind] = actualTrace
error[ind] = err
# testsPool['T1'] = [0,3] #TODO: delete
# testsPool['T3'] = [2,3]
old = None
if cut:
old = components, initials
testsPool, error, components, initials = cut_matrix(
testsPool, error, components, initials)
Experiment_Data().set_values(priors, bugs, testsPool, components,
estimatedTestsPool)
diagnoser = partial(
software.sfl_diagnoser.Diagnoser.ExperimentInstance.ExperimentInstance,
initials)
smart_mhs_diagnoser = FastBarinel(initials, error)
return diagnoser, smart_mhs_diagnoser, error, initials, old
def components_diagnoses(ei, aggregation):
ei.diagnose()
optionals = ei.get_optionals_actions()
probabilities = []
components = {}
for d in ei.diagnoses:
for c in d.get_diag():
components[c] = components.get(c, 0) + 1
for test in optionals:
trace = Experiment_Data().POOL[test]
probabilities.append(aggregation([components.get(c, 0)
for c in trace]))
return [x / sum(probabilities) for x in probabilities]
def pipeline(folder, output):
second_folder = folder + '_reducted'
matrices = [
file for file in os.listdir(folder) if file.endswith('.matrix')
]
matrices = sorted(matrices,
key=file2tuple) # for logic traversal of matrices
print(f'total of {len(matrices)} matrices')
matrices = list(filter(filter_low, matrices))
# matrices = list(filter(filter_low, matrices))
# from_matrix = '14_14_11.matrix'
# from_index = dict(map(reversed,enumerate(matrices)))[from_matrix]
# to_matrix = '14_14_20.matrix'
# to_index = dict(map(reversed,enumerate(matrices)))[to_matrix]
# matrices = matrices[from_index:to_index + 1]
print(f'remaining {len(matrices)} matrices')
# print(matrices[0])
# return
for matrix_file in matrices:
print(f'diagnosing matrix {matrix_file}')
# matrix_file = '1_2_1.matrix'
inst, error, initials, _ = readPlanningFile(os.path.join(
folder, matrix_file),
cut=True)
# uncertain_tests = Experiment_Data().POOL.keys()
uncertain_tests = [
test for (test, outcome) in error.items() if outcome == 1
]
alg1_result, alg1_time, _, _ = run_diagnoser(inst, error, 0.1,
uncertain_tests)
inst, error, initials, _ = readPlanningFile(os.path.join(
second_folder, matrix_file),
cut=True)
if len(Experiment_Data().COMPONENTS_NAMES) <= 2:
print('small matrix. continuing')
continue
alg2_result, alg2_time, _, _ = run_reduction_based(
inst, error, 0.1, uncertain_tests)
alg1_diags = list(map(operator.itemgetter(0), alg1_result))
alg2_diags = list(map(operator.itemgetter(0), alg2_result))
if alg1_diags != alg2_diags:
print(f'difference in {matrix_file}:')
print(f'o2d results: {alg1_result}')
print(f'reduction results: {alg2_result}')
print_matrix(error)
break
def next_tests_by_prob(self):
"""
order tests by probabilities of the components
return tests and probabilities
"""
compsProbs = self.get_components_probabilities()
comps_probabilities = dict(compsProbs)
optionals = self.get_optionals_actions()
assert len(optionals) > 0
tests_probabilities = []
for test in optionals:
trace = Experiment_Data().ESTIMATED_POOL[test]
test_p = 0.0
for comp in trace:
test_p += comps_probabilities.get(comp, 0) * trace.get(comp, 0)
tests_probabilities.append(test_p)
if sum(tests_probabilities) == 0.0:
return self.get_optionals_probabilities()
tests_probabilities = [abs(x) for x in tests_probabilities]
tests_probabilities = [
x / sum(tests_probabilities) for x in tests_probabilities
]
return optionals, tests_probabilities
def next_tests_by_bd(self):
self.diagnose()
probabilities = []
optionals = self.get_optionals_actions()
for test in optionals:
p = 0.0
trace = Experiment_Data().POOL[test]
for d in self.diagnoses:
p += (d.get_prob() / len(d.get_diag())) * (
[x for x in d.get_diag() if x in trace])
probabilities.append(p)
probabilities = [abs(x) for x in probabilities]
probabilities = [x / sum(probabilities) for x in probabilities]
return optionals, probabilities
def write_merged_matrix(instance, out_matrix):
componets = instance.get_components_vectors()
similiar_componets = {}
for component in componets:
similiar_componets.setdefault(str(sorted(componets[component])),
[]).append(component)
new_components_map = {}
for comp in similiar_componets:
candidates = similiar_componets[comp]
new_name = "^".join(similiar_componets[comp])
for candidate in candidates:
new_components_map[candidate] = new_name
get_name = lambda index: new_components_map.get(
Experiment_Data().COMPONENTS_NAMES[index],
Experiment_Data().COMPONENTS_NAMES[index])
new_bugs = map(get_name, Experiment_Data().BUGS)
new_pool = map(
lambda test: [
test,
list(set(map(get_name,
Experiment_Data().POOL[test]))), instance.error[test]
],
Experiment_Data().POOL)
write_planning_file(out_matrix, new_bugs, new_pool)
def diagnose_smart_mhs(diagnoser, faulty_comp_prob, faulty_output_prob):
components_dict = Experiment_Data().COMPONENTS_NAMES
unseen_prob_sum = 1
diagnoses = {}
for obs_diags, obs_prob in diagnoser.diagnose(faulty_comp_prob, faulty_output_prob, normalize=False):
for diag in obs_diags:
comps = tuple(sorted([components_dict[c] for c in diag.diagnosis]))
diagnoses[comps] = diagnoses.get(comps, 0) + diag.probability * obs_prob
unseen_prob_sum -= obs_prob
max_prob = max(diagnoses.values())
second_max_prob = max(list(filter(lambda p: p < max_prob, diagnoses.values())) + [0])
if max_prob - second_max_prob > unseen_prob_sum:
return [(d, p) for d, p in diagnoses.items() if p == max_prob]
return [(d,p) for d,p in diagnoses.items() if p == max(diagnoses.values())]
def childs_probs_by_hp(self):
"""
compute HP for the optionals tests and return dict of (test, prob)
"""
comps_prob = dict(
self.get_components_probabilities()) # tuples of (comp, prob)
optionals = self.get_optionals_actions()
assert len(optionals) > 0
optionals_probs = {}
for op in optionals:
trace = Experiment_Data().POOL[op]
prob = 0
for comp in trace:
prob += comps_prob.get(comp, 0)
optionals_probs[op] = prob
return optionals_probs
def diagnose_all_combinations(inst, error, faulty_output_prob,
uncertain_tests):
components_dict = Experiment_Data().COMPONENTS_NAMES
# sum_probs = diagnoser_utils.obs_normalization(len(uncertain_tests), faulty_output_prob)
diagnoses = {}
diagnoser = inst(error)
diagnoser.diagnose()
for diag in diagnoser.diagnoses:
comps = sorted([components_dict[c] for c in diag.diagnosis])
filtered_comps = tuple(comp for comp in comps if comp.startswith('c'))
# num_of_test_in_diag = len(comps) - len(filtered_comps)
# obs_prob = pow(faulty_output_prob,num_of_test_in_diag) / sum_probs
old_diag_prob = diagnoses.get(filtered_comps, 0)
diagnoses[filtered_comps] = old_diag_prob + diag.probability
return diagnoses.items()
def print_matrix(error):
comps = Experiment_Data().COMPONENTS_NAMES
comps_names = sorted(comps.values())
reverse_comps_dict = dict([reversed(t) for t in comps.items()])
trace = Experiment_Data().POOL
tests_names = sorted(trace.keys())
print('\t\\ \t|\t' + '\t|\t'.join(comps_names) + '\t|\te\t|')
for test in tests_names:
touching = [
1 if reverse_comps_dict[c] in trace[test] else 0
for c in comps_names
]
# buggy = 1 if test in error else 0
print(f'\t{test}\t|\t' + '\t|\t '.join(map(str, touching)) +
f'\t|\t{error[test]}\t|')
