def random_sampling(simulations: int, mcts_stats_its):
print("Problem 1 (simulated): Finding defective configurations in the jHipster feature model.")
print("--------------------------------------------------------------------------------------")
print("Setting up the problem...")
print("Creating output folders...")
if not os.path.exists(OUTPUT_RESULTS_PATH):
os.makedirs(OUTPUT_RESULTS_PATH)
if not os.path.exists(HEATMAP_PATH):
os.makedirs(HEATMAP_PATH)
if not os.path.exists(STATS_PATH):
os.makedirs(STATS_PATH)
print(f"Loading feature model: {jhipster.FM_FILE} ...")
fide_parser = FeatureIDEParser(jhipster.FM_FILE, no_read_constraints=True)
fm = fide_parser.transform()
print(f"Feature model loaded with {len(fm.get_features())} features, {len(fm.get_constraints())} constraints, {len(fm.get_relations())} relations.")
# Read the feature model as CNF model with complex constraints
cnf_reader = CNFReader(jhipster.CNF_FILE)
cnf_model = cnf_reader.transform()
total_time_start = time.time()
sample, n_positive_evaluations = jhipster.get_random_sampling(simulations)
total_time_end = time.time()
mcts_stats_its.add_step('Random Sampling', 0, {}, simulations, len(sample), n_positive_evaluations, total_time_end-total_time_start)
def calculate_montecarlo_approximation():
fide_parser = FeatureIDEParser(jhipster.FM_FILE, no_read_constraints=True)
fm = fide_parser.transform()
# Read the feature model as CNF model with complex constraints
cnf_reader = CNFReader(jhipster.CNF_FILE)
cnf_model = cnf_reader.transform()
# Read the jHipster configurations
jhipster_configurations = jhipster.read_jHipster_feature_model_configurations()
bdd = BDDModel(feature_model=fm, cnf_formula=JHIPSTER)
nof_configurations = bdd.get_number_of_configurations()
all_configurations = bdd.get_configurations()
all_defective_configs = [c for c in all_configurations if jhipster_configurations[c]]
all_prob_defect_configs = round(len(all_defective_configs) / len(all_configurations), DIGIT_PRECISION)
print(f"Variables: {bdd.variables}")
print(f"#Variables: {len(bdd.variables)}")
print(f"#Configurations: {len(all_configurations)}")
with open("jhipster_mc_approximation.csv", 'w+') as file:
file.write("Runs, Total configs., Total Defective configs., Real Prob. Defect. configs., Percentage, Simulations, Median defect. configs., Mean defect. configs., Std defect. configs., Prob. defect. configs., Error\n")
percentages = [x/1000 for x in range(1, 101, 1)]
simulations_list = [1] + [x for x in range(10, 5001, 10)]
#for percentage_sim in percentages:
print("Simulation: ", end='', flush=True)
for simulations in simulations_list:
runs = RUNS
#simulations = math.ceil(len(all_configurations) * percentage_sim)
percentage_sim = round(simulations/len(all_configurations), DIGIT_PRECISION)
print(f"{simulations} ", end='', flush=True)
defective_configs_in_sample_per_runs = []
for r in range(runs):
sample_configurations = random.sample(all_configurations, simulations)
defective_configs_in_sample = [c for c in sample_configurations if jhipster_configurations[c]]
defective_configs_in_sample_per_runs.append(len(defective_configs_in_sample))
median_defective_configs_in_sample = round(statistics.median(defective_configs_in_sample_per_runs), DIGIT_PRECISION)
mean_defective_configs_in_sample = round(statistics.mean(defective_configs_in_sample_per_runs), DIGIT_PRECISION)
std_defective_configs_in_sample = round(statistics.stdev(defective_configs_in_sample_per_runs), DIGIT_PRECISION)
sample_median_probability_defective_configs = round(median_defective_configs_in_sample / simulations, DIGIT_PRECISION)
error = round(abs(sample_median_probability_defective_configs - all_prob_defect_configs), DIGIT_PRECISION)
file.write(f'{runs}, {len(all_configurations)}, {len(all_defective_configs)}, {all_prob_defect_configs}, {percentage_sim}, {simulations}, ' \
+ f'{median_defective_configs_in_sample}, {mean_defective_configs_in_sample}, {std_defective_configs_in_sample}, {sample_median_probability_defective_configs}, {error}\n')
def random_sampling(simulations: int, input_fm_name: str,
input_fm_cnf_name: str, mcts_stats_its):
print(
"Problem 1 (Random sampling): Finding defective configurations in the AAFMs Python Framework feature model."
)
print(
"----------------------------------------------------------------------------------------------------"
)
print("Setting up the problem...")
print("Creating output folders...")
if not os.path.exists(OUTPUT_RESULTS_PATH):
os.makedirs(OUTPUT_RESULTS_PATH)
if not os.path.exists(HEATMAP_PATH):
os.makedirs(HEATMAP_PATH)
if not os.path.exists(STATS_PATH):
os.makedirs(STATS_PATH)
input_fm = INPUT_PATH + input_fm_name + ".xml"
print(f"Loading feature model: {input_fm_name} ...")
fide_parser = FeatureIDEParser(input_fm, no_read_constraints=True)
fm = fide_parser.transform()
print(
f"Feature model loaded with {len(fm.get_features())} features, {len(fm.get_constraints())} constraints, {len(fm.get_relations())} relations."
)
# Read the feature model as CNF model with complex constraints
cnf_reader = CNFReader(INPUT_PATH + input_fm_cnf_name + ".txt")
cnf_model = cnf_reader.transform()
# AAFMs
aafms_helper = AAFMsHelper(fm, cnf_model)
all_configurations = aafms_helper.get_configurations()
total_time_start = time.time()
if simulations > len(all_configurations):
simulations = len(all_configurations)
sample = random.sample(all_configurations, simulations)
defective_configurations = [c for c in sample if count_errors(fm, c) > 0]
total_time_end = time.time()
mcts_stats_its.add_step('Random Sampling', 0, {}, simulations, len(sample),
len(defective_configurations),
total_time_end - total_time_start)
def main():
fide_parser = FeatureIDEParser(jhipster.FM_FILE, no_read_constraints=True)
fm = fide_parser.transform()
# Read the feature model as CNF model with complex constraints
cnf_reader = CNFReader(jhipster.CNF_FILE)
cnf_model = cnf_reader.transform()
# Read the jHipster configurations
jhipster_configurations = jhipster.read_jHipster_feature_model_configurations()
bdd = BDDModel(feature_model=fm, cnf_formula=JHIPSTER)
print(f"Variables: {bdd.variables}")
print(f"#Variables: {len(bdd.variables)}")
print(f"#Configurations: {bdd.get_number_of_configurations()}")
vp_analysis = VariationPointAnalysis(fm, bdd)
vps = vp_analysis.get_variation_points()
print(f"#Variation points: {len(vps)}")
# for vp in vps:
# print(f"VP: {vp}")
# combinations = vp_analysis.get_variants_combinations(vp)
# print(f" |->#{len(combinations)}")
with open("jhipster_analysis.csv", 'w+') as file:
file.write("Variation points, Variation point, Configurations, Defective configs., Real prob. defect. configs., Simulations, Median defect. configs., Mean defect. configs., Std defect. configs., Prob. defect. configs., Combinations, Variant combination, NofFeatures, Configs. combi., Ratio, Defect. configs. combi., Prob. defect. configs. combi., Simulations, Runs, Median defect. configs., Mean defect. configs., Std defect. configs., Prob. defect. configs.\n")
for vp in vps:
# Real probabilities of each variation point
vp_configurations = bdd.get_configurations(selected_features=[vp])
vp_defective_configs = [c for c in vp_configurations if jhipster_configurations[c]]
vp_probability_defective_configs = round(len(vp_defective_configs) / len(vp_configurations), DIGIT_PRECISION)
# Monte Carlo simulations for the variation point
vp_simulations = math.ceil(len(vp_configurations) * PERCENTAGE_SIMULATIONS)
runs = RUNS
vp_defective_configs_in_sample_per_runs = []
for r in range(runs):
vp_sample_configurations = random.sample(vp_configurations, vp_simulations)
vp_defective_configs_in_sample = [c for c in vp_sample_configurations if jhipster_configurations[c]]
vp_defective_configs_in_sample_per_runs.append(len(vp_defective_configs_in_sample))
vp_median_defective_configs_in_sample = round(statistics.median(vp_defective_configs_in_sample_per_runs), DIGIT_PRECISION)
vp_mean_defective_configs_in_sample = round(statistics.mean(vp_defective_configs_in_sample_per_runs), DIGIT_PRECISION)
vp_std_defective_configs_in_sample = round(statistics.stdev(vp_defective_configs_in_sample_per_runs), DIGIT_PRECISION)
vp_sample_median_probability_defective_configs = round(vp_median_defective_configs_in_sample / vp_simulations, DIGIT_PRECISION)
combinations = vp_analysis.get_variants_combinations(vp)
variants = vp_analysis.get_variants(vp)
combinations = variants # only for optional features analysis
optional_choice = [True, False]
for combi in combinations:
# Real probabilities of each variant combination
#selected_features = list(set(combi).union({vp}))
#deselected_features = list(set(variants) - set(combi))
# only for optional features analysis
original_combi = combi
for choice in optional_choice:
if choice:
combi = [original_combi]
selected_features = combi + [vp]
deselected_features = []
else:
combi = []
selected_features = [vp]
deselected_features = [original_combi]
combi_configurations = bdd.get_configurations(selected_features=selected_features, deselected_features=deselected_features)
combi_configurations_ratio = round(len(combi_configurations) / len(vp_configurations), DIGIT_PRECISION)
if len(combi_configurations) > 0:
combi_defective_configs = [c for c in combi_configurations if jhipster_configurations[c]]
combi_probability_defective_configs = round(len(combi_defective_configs) / len(combi_configurations), DIGIT_PRECISION)
# Monte Carlo simulations
simulations = math.ceil(len(combi_configurations) * PERCENTAGE_SIMULATIONS)
# if simulations > len(combi_configurations):
# simulations = len(combi_configurations)
runs = RUNS
defective_configs_in_sample_per_runs = []
for r in range(runs):
sample_configurations = random.sample(combi_configurations, simulations)
defective_configs_in_sample = [c for c in sample_configurations if jhipster_configurations[c]]
defective_configs_in_sample_per_runs.append(len(defective_configs_in_sample))
median_defective_configs_in_sample = round(statistics.median(defective_configs_in_sample_per_runs), DIGIT_PRECISION)
mean_defective_configs_in_sample = round(statistics.mean(defective_configs_in_sample_per_runs), DIGIT_PRECISION)
std_defective_configs_in_sample = round(statistics.stdev(defective_configs_in_sample_per_runs), DIGIT_PRECISION)
sample_median_probability_defective_configs = round(median_defective_configs_in_sample / simulations, DIGIT_PRECISION)
file.write(f'{len(vps)}, {vp.name}, {len(vp_configurations)}, {len(vp_defective_configs)}, {vp_probability_defective_configs}, ' \
+ f'{vp_simulations}, {vp_median_defective_configs_in_sample}, {vp_mean_defective_configs_in_sample}, {vp_std_defective_configs_in_sample}, {vp_sample_median_probability_defective_configs}, ' \
+ f'{len(combinations)}, "{[str(f) for f in combi]}", {len(combi)}, {len(combi_configurations)}, {combi_configurations_ratio}, {len(combi_defective_configs)}, {combi_probability_defective_configs}, ' \
+ f'{simulations}, {runs}, {median_defective_configs_in_sample}, {mean_defective_configs_in_sample}, {std_defective_configs_in_sample}, {sample_median_probability_defective_configs}\n')
else:
file.write(f'{len(vps)}, {vp.name}, {len(vp_configurations)}, {len(vp_defective_configs)}, {vp_probability_defective_configs}, ' \
+ f'{vp_simulations}, {vp_median_defective_configs_in_sample}, {vp_mean_defective_configs_in_sample}, {vp_std_defective_configs_in_sample}, {vp_sample_median_probability_defective_configs}, ' \
+ f'{len(combinations)}, "{[str(f) for f in combi]}", {len(combi)}, {len(combi_configurations)}, {combi_configurations_ratio}, {0}, {0}, ' \
+ f'{"-"}, {"-"}, {"-"}, {"-"}, {"-"}, {"-"}\n')
def main(algorithm, simulations: int, mcts_stats_its):
print("Problem 1 (simulated): Finding defective configurations in the jHipster feature model.")
print("--------------------------------------------------------------------------------------")
print("Setting up the problem...")
print("Creating output folders...")
if not os.path.exists(OUTPUT_RESULTS_PATH):
os.makedirs(OUTPUT_RESULTS_PATH)
if not os.path.exists(HEATMAP_PATH):
os.makedirs(HEATMAP_PATH)
if not os.path.exists(STATS_PATH):
os.makedirs(STATS_PATH)
print(f"Loading feature model: {jhipster.FM_FILE} ...")
fide_parser = FeatureIDEParser(jhipster.FM_FILE, no_read_constraints=True)
fm = fide_parser.transform()
print(f"Feature model loaded with {len(fm.get_features())} features, {len(fm.get_constraints())} constraints, {len(fm.get_relations())} relations.")
# Read the feature model as CNF model with complex constraints
cnf_reader = CNFReader(jhipster.CNF_FILE)
cnf_model = cnf_reader.transform()
# AAFMs
aafms_helper = AAFMsHelper(fm, cnf_model)
print(f"Creating set of actions...")
actions = TreeActionsList(fm)
print(f"{actions.get_nof_actions()} actions.")
problem_data = ProblemData(fm, aafms_helper, actions)
# Read the jhipster configurations as a dict of FMConfiguration -> bool (failure)
jhipster_configurations = jhipster.read_jHipster_feature_model_configurations()
problem_data.jhipster_configurations = jhipster_configurations
problem_data.sample = defaultdict(bool)
print(f"Creating initial state (configuration)...")
initial_config = FMConfiguration()
initial_state = FailureConfigurationState(configuration=initial_config, data=problem_data)
print(f"Initial state: {initial_state}")
print("Problem setted up.")
print(f"Running algorithm {str(algorithm)}...")
# Stats
mcts_stats = MCTSStats()
n = 0
total_evaluations = 0
state = initial_state
total_time_start = time.time()
while state.reward() <= 0 and state.get_actions():
print(f"Input state {n}: {str(state)} -> valid={state.is_valid_configuration}, R={state.reward()}")
time_start = time.time()
new_state = algorithm.run(state)
time_end = time.time()
if isinstance(algorithm, MonteCarloTreeSearch):
# Heat map (only for MCTS)
heatmap = Heatmap(fm, algorithm.tree, algorithm.Q, algorithm.N, state)
heatmap.extract_feature_knowledge()
heatmap.serialize(HEATMAP_PATH + jhipster.FM_FILENAME + "-step" + str(n) + ".csv")
else:
algorithm.tree = {}
# Stats
mcts_stats.add_step(n, algorithm.tree, state, new_state, simulations, algorithm.n_evaluations, algorithm.n_positive_evaluations, time_end-time_start)
total_evaluations += algorithm.n_evaluations
algorithm.n_evaluations = 0
state = new_state
n += 1
total_time_end = time.time()
print("Algorithm finished.")
print(f"Final state {n}: {str(state)} -> valid={state.is_valid_configuration}, R={state.reward()}")
# Stats
print("Serializing results...")
mcts_stats.serialize(STATS_PATH + jhipster.FM_FILENAME + '-steps.csv')
mcts_stats_its.add_step(str(algorithm), n, algorithm.tree, simulations, total_evaluations, algorithm.n_positive_evaluations, total_time_end-total_time_start)
#mcts_stats_its.serialize(STATS_PATH + jhipster.FM_FILENAME + '-summary.csv')
print("Done!")
def main(algorithm, simulations: int, input_fm: str, input_cnf_model: str=None):
print("Problem: Reverse engineering of feature models.")
print("-----------------------------------------------")
base = os.path.basename(input_fm)
input_fm_name = os.path.splitext(base)[0]
print("Setting up the problem...")
print("Creating output folders...")
if not os.path.exists(HEATMAP_PATH):
os.makedirs(HEATMAP_PATH)
if not os.path.exists(STATS_PATH):
os.makedirs(STATS_PATH)
if not os.path.exists(GENERATED_FMS_PATH):
os.makedirs(GENERATED_FMS_PATH)
print(f"Loading feature model: {input_fm_name} ...")
fide_parser = FeatureIDEParser(input_fm, no_read_constraints=(input_cnf_model is not None))
fm = fide_parser.transform()
print(f"Feature model loaded with {len(fm.get_features())} features, {len(fm.get_constraints())} constraints, {len(fm.get_relations())} relations.")
if input_cnf_model is not None:
# Read the feature model as CNF model with complex constraints
cnf_reader = CNFReader(input_cnf_model)
cnf_model = cnf_reader.transform()
else:
cnf_model = None
# Get configurations
print("Generating configurations of the feature model...")
aafms_helper = AAFMsHelper(fm, cnf_model)
configurations = aafms_helper.get_configurations()
print(f"#Configurations: {len(configurations)}")
print(f"Creating initial state (empty feature model)...")
initial_state = FMState(FeatureModel(None), configurations)
print("Problem setted up.")
print(f"Running algorithm {str(algorithm)}...")
# Stats
mcts_stats_re = MCTSStatsRE(STATS_PATH + input_fm_name + "-ReverseEngineering.log")
n = 0
state = initial_state
total_time_start = time.time()
while not state.is_terminal():
print(f"State {n}: {[str(f) for f in state.feature_model.get_features()]} -> {state.reward()}")
start_time = time.time()
new_state = algorithm.run(state)
end_time = time.time()
if isinstance(algorithm, MonteCarloTreeSearch):
mcts_stats_re.add_step(n, algorithm.tree, algorithm.Q, algorithm.N, state, new_state, simulations, end_time-start_time)
else:
algorithm.tree = {}
state = new_state
n += 1
total_time_end = time.time()
print("Algorithm finished.")
print(f"Final State {n}: {[str(f) for f in state.feature_model.get_features()]} -> {state.reward()}")
# Get configurations
path = GENERATED_FMS_PATH + state.feature_model.root.name + "." + UVLWritter.get_destination_extension()
print(f"Serializing generated feature model in UVL format in {path}")
uvl_writter = UVLWritter(path=path, source_model=state.feature_model)
uvl_writter.transform()
# Get configurations
print("Generating configurations of the extracted feature model...")
aafms_helper = AAFMsHelper(state.feature_model)
new_configurations = aafms_helper.get_configurations()
print("Results:")
print(f"#Features: {len(state.feature_model.get_features())} -> {[str(f) for f in state.feature_model.get_features()]}")
print(f"#Configurations: {len(new_configurations)}")
relaxed_value = reduce(lambda count, c: count + (aafms_helper.is_valid_configuration(c)), configurations, 0)
deficit_value = reduce(lambda count, c: count + (c not in new_configurations), configurations, 0)
surplus_value = reduce(lambda count, c: count + (c not in configurations), new_configurations, 0)
print(f"Input configurations captured (Relaxed objective function): {relaxed_value}")
print(f"Deficit of configurations: {deficit_value}")
print(f"Irrelevant configurations: {surplus_value}")
print(f"Mininal difference (MinDiff) objective function (deficit_value + surplus_value): {deficit_value} + {surplus_value} = {deficit_value+surplus_value}")
print(f"Final objective function (Relaxed - MinDiff): {relaxed_value - (deficit_value+surplus_value)}")
print(f"Execution time: {total_time_end-total_time_start}")
def main(algorithm,
simulations: int,
input_fm: str,
input_cnf_model: str = None,
initial_config_features: list[str] = [],
minimum: bool = False):
print("Problem: Completion of partial configurations.")
print("----------------------------------------------")
print("Setting up the problem...")
print("Creating output folders...")
if not os.path.exists(OUTPUT_RESULTS_PATH):
os.makedirs(OUTPUT_RESULTS_PATH)
if not os.path.exists(HEATMAP_PATH):
os.makedirs(HEATMAP_PATH)
if not os.path.exists(STATS_PATH):
os.makedirs(STATS_PATH)
base = os.path.basename(input_fm)
input_fm_name = os.path.splitext(base)[0]
print(f"Loading feature model: {input_fm} ...")
fide_parser = FeatureIDEParser(input_fm,
no_read_constraints=(input_cnf_model
is not None))
fm = fide_parser.transform()
print(
f"Feature model loaded with {len(fm.get_features())} features, {len(fm.get_constraints())} constraints, {len(fm.get_relations())} relations."
)
if input_cnf_model is not None:
# Read the feature model as CNF model with complex constraints
cnf_reader = CNFReader(input_cnf_model)
cnf_model = cnf_reader.transform()
else:
cnf_model = None
# AAFMs
aafms_helper = AAFMsHelper(fm, cnf_model)
print(f"Creating set of actions...")
actions = TreeActionsList(fm)
print(f"{actions.get_nof_actions()} actions.")
problem_data = ProblemData(fm, aafms_helper, actions)
print(f"Creating initial state (configuration)...")
if initial_config_features:
elements = {}
for feature in [
fm.get_feature_by_name(f) for f in initial_config_features
]:
elements[feature] = True
for p in fm_utils.select_parent_features(feature):
elements[p] = True
initial_config = FMConfiguration(elements=elements)
else:
initial_config = FMConfiguration()
if minimum:
initial_state = ValidMinimumConfigurationState(
configuration=initial_config, data=problem_data)
else:
initial_state = ValidConfigurationState(configuration=initial_config,
data=problem_data)
print(f"Initial state: {initial_state}")
print("Problem setted up.")
print(f"Running algorithm {str(algorithm)}...")
# Stats
mcts_stats = MCTSStats()
mcts_stats_its = MCTSStatsIts()
n = 0
total_evaluations = 0
state = initial_state
total_time_start = time.time()
while not state.is_terminal():
print(
f"Input state {n}: {str(state)} -> valid={state.is_valid_configuration}, R={state.reward()}"
)
time_start = time.time()
new_state = algorithm.run(state)
time_end = time.time()
if isinstance(algorithm, MonteCarloTreeSearch):
# Heat map (only for MCTS)
heatmap = Heatmap(fm, algorithm.tree, algorithm.Q, algorithm.N,
state)
heatmap.extract_feature_knowledge()
heatmap.serialize(HEATMAP_PATH + input_fm_name + "-step" + str(n) +
".csv")
else:
algorithm.tree = {}
# Stats
mcts_stats.add_step(n, algorithm.tree, state, new_state, simulations,
algorithm.n_evaluations,
algorithm.n_positive_evaluations,
time_end - time_start)
total_evaluations += algorithm.n_evaluations
algorithm.n_evaluations = 0
state = new_state
n += 1
total_time_end = time.time()
print("Algorithm finished.")
print(
f"Final state {n}: {str(state)} -> valid={state.is_valid_configuration}, R={state.reward()}"
)
# Stats
print("Serializing results...")
mcts_stats.serialize(STATS_PATH + input_fm_name + '-steps.csv')
mcts_stats_its.add_step(str(algorithm), n, algorithm.tree, simulations,
total_evaluations,
algorithm.n_positive_evaluations,
total_time_end - total_time_start)
mcts_stats_its.serialize(STATS_PATH + input_fm_name + '-summary.csv')
# Heat for the whole process (not sure about its utility)
if isinstance(algorithm, MonteCarloTreeSearch):
heatmap = HeatmapFull(fm, algorithm.tree, algorithm.Q, algorithm.N)
heatmap.extract_feature_knowledge()
heatmap.serialize(HEATMAP_PATH + input_fm_name + "-full.csv")
print("Done!")