def random_sampling(simulations: int, mcts_stats_its): print("Problem 1 (simulated): Finding defective configurations in the jHipster feature model.") print("--------------------------------------------------------------------------------------") print("Setting up the problem...") print("Creating output folders...") if not os.path.exists(OUTPUT_RESULTS_PATH): os.makedirs(OUTPUT_RESULTS_PATH) if not os.path.exists(HEATMAP_PATH): os.makedirs(HEATMAP_PATH) if not os.path.exists(STATS_PATH): os.makedirs(STATS_PATH) print(f"Loading feature model: {jhipster.FM_FILE} ...") fide_parser = FeatureIDEParser(jhipster.FM_FILE, no_read_constraints=True) fm = fide_parser.transform() print(f"Feature model loaded with {len(fm.get_features())} features, {len(fm.get_constraints())} constraints, {len(fm.get_relations())} relations.") # Read the feature model as CNF model with complex constraints cnf_reader = CNFReader(jhipster.CNF_FILE) cnf_model = cnf_reader.transform() total_time_start = time.time() sample, n_positive_evaluations = jhipster.get_random_sampling(simulations) total_time_end = time.time() mcts_stats_its.add_step('Random Sampling', 0, {}, simulations, len(sample), n_positive_evaluations, total_time_end-total_time_start)
def calculate_montecarlo_approximation(): fide_parser = FeatureIDEParser(jhipster.FM_FILE, no_read_constraints=True) fm = fide_parser.transform() # Read the feature model as CNF model with complex constraints cnf_reader = CNFReader(jhipster.CNF_FILE) cnf_model = cnf_reader.transform() # Read the jHipster configurations jhipster_configurations = jhipster.read_jHipster_feature_model_configurations() bdd = BDDModel(feature_model=fm, cnf_formula=JHIPSTER) nof_configurations = bdd.get_number_of_configurations() all_configurations = bdd.get_configurations() all_defective_configs = [c for c in all_configurations if jhipster_configurations[c]] all_prob_defect_configs = round(len(all_defective_configs) / len(all_configurations), DIGIT_PRECISION) print(f"Variables: {bdd.variables}") print(f"#Variables: {len(bdd.variables)}") print(f"#Configurations: {len(all_configurations)}") with open("jhipster_mc_approximation.csv", 'w+') as file: file.write("Runs, Total configs., Total Defective configs., Real Prob. Defect. configs., Percentage, Simulations, Median defect. configs., Mean defect. configs., Std defect. configs., Prob. defect. configs., Error\n") percentages = [x/1000 for x in range(1, 101, 1)] simulations_list = [1] + [x for x in range(10, 5001, 10)] #for percentage_sim in percentages: print("Simulation: ", end='', flush=True) for simulations in simulations_list: runs = RUNS #simulations = math.ceil(len(all_configurations) * percentage_sim) percentage_sim = round(simulations/len(all_configurations), DIGIT_PRECISION) print(f"{simulations} ", end='', flush=True) defective_configs_in_sample_per_runs = [] for r in range(runs): sample_configurations = random.sample(all_configurations, simulations) defective_configs_in_sample = [c for c in sample_configurations if jhipster_configurations[c]] defective_configs_in_sample_per_runs.append(len(defective_configs_in_sample)) median_defective_configs_in_sample = round(statistics.median(defective_configs_in_sample_per_runs), DIGIT_PRECISION) mean_defective_configs_in_sample = round(statistics.mean(defective_configs_in_sample_per_runs), DIGIT_PRECISION) std_defective_configs_in_sample = round(statistics.stdev(defective_configs_in_sample_per_runs), DIGIT_PRECISION) sample_median_probability_defective_configs = round(median_defective_configs_in_sample / simulations, DIGIT_PRECISION) error = round(abs(sample_median_probability_defective_configs - all_prob_defect_configs), DIGIT_PRECISION) file.write(f'{runs}, {len(all_configurations)}, {len(all_defective_configs)}, {all_prob_defect_configs}, {percentage_sim}, {simulations}, ' \ + f'{median_defective_configs_in_sample}, {mean_defective_configs_in_sample}, {std_defective_configs_in_sample}, {sample_median_probability_defective_configs}, {error}\n')
def random_sampling(simulations: int, input_fm_name: str, input_fm_cnf_name: str, mcts_stats_its): print( "Problem 1 (Random sampling): Finding defective configurations in the AAFMs Python Framework feature model." ) print( "----------------------------------------------------------------------------------------------------" ) print("Setting up the problem...") print("Creating output folders...") if not os.path.exists(OUTPUT_RESULTS_PATH): os.makedirs(OUTPUT_RESULTS_PATH) if not os.path.exists(HEATMAP_PATH): os.makedirs(HEATMAP_PATH) if not os.path.exists(STATS_PATH): os.makedirs(STATS_PATH) input_fm = INPUT_PATH + input_fm_name + ".xml" print(f"Loading feature model: {input_fm_name} ...") fide_parser = FeatureIDEParser(input_fm, no_read_constraints=True) fm = fide_parser.transform() print( f"Feature model loaded with {len(fm.get_features())} features, {len(fm.get_constraints())} constraints, {len(fm.get_relations())} relations." ) # Read the feature model as CNF model with complex constraints cnf_reader = CNFReader(INPUT_PATH + input_fm_cnf_name + ".txt") cnf_model = cnf_reader.transform() # AAFMs aafms_helper = AAFMsHelper(fm, cnf_model) all_configurations = aafms_helper.get_configurations() total_time_start = time.time() if simulations > len(all_configurations): simulations = len(all_configurations) sample = random.sample(all_configurations, simulations) defective_configurations = [c for c in sample if count_errors(fm, c) > 0] total_time_end = time.time() mcts_stats_its.add_step('Random Sampling', 0, {}, simulations, len(sample), len(defective_configurations), total_time_end - total_time_start)
def main(): fide_parser = FeatureIDEParser(jhipster.FM_FILE, no_read_constraints=True) fm = fide_parser.transform() # Read the feature model as CNF model with complex constraints cnf_reader = CNFReader(jhipster.CNF_FILE) cnf_model = cnf_reader.transform() # Read the jHipster configurations jhipster_configurations = jhipster.read_jHipster_feature_model_configurations() bdd = BDDModel(feature_model=fm, cnf_formula=JHIPSTER) print(f"Variables: {bdd.variables}") print(f"#Variables: {len(bdd.variables)}") print(f"#Configurations: {bdd.get_number_of_configurations()}") vp_analysis = VariationPointAnalysis(fm, bdd) vps = vp_analysis.get_variation_points() print(f"#Variation points: {len(vps)}") # for vp in vps: # print(f"VP: {vp}") # combinations = vp_analysis.get_variants_combinations(vp) # print(f" |->#{len(combinations)}") with open("jhipster_analysis.csv", 'w+') as file: file.write("Variation points, Variation point, Configurations, Defective configs., Real prob. defect. configs., Simulations, Median defect. configs., Mean defect. configs., Std defect. configs., Prob. defect. configs., Combinations, Variant combination, NofFeatures, Configs. combi., Ratio, Defect. configs. combi., Prob. defect. configs. combi., Simulations, Runs, Median defect. configs., Mean defect. configs., Std defect. configs., Prob. defect. configs.\n") for vp in vps: # Real probabilities of each variation point vp_configurations = bdd.get_configurations(selected_features=[vp]) vp_defective_configs = [c for c in vp_configurations if jhipster_configurations[c]] vp_probability_defective_configs = round(len(vp_defective_configs) / len(vp_configurations), DIGIT_PRECISION) # Monte Carlo simulations for the variation point vp_simulations = math.ceil(len(vp_configurations) * PERCENTAGE_SIMULATIONS) runs = RUNS vp_defective_configs_in_sample_per_runs = [] for r in range(runs): vp_sample_configurations = random.sample(vp_configurations, vp_simulations) vp_defective_configs_in_sample = [c for c in vp_sample_configurations if jhipster_configurations[c]] vp_defective_configs_in_sample_per_runs.append(len(vp_defective_configs_in_sample)) vp_median_defective_configs_in_sample = round(statistics.median(vp_defective_configs_in_sample_per_runs), DIGIT_PRECISION) vp_mean_defective_configs_in_sample = round(statistics.mean(vp_defective_configs_in_sample_per_runs), DIGIT_PRECISION) vp_std_defective_configs_in_sample = round(statistics.stdev(vp_defective_configs_in_sample_per_runs), DIGIT_PRECISION) vp_sample_median_probability_defective_configs = round(vp_median_defective_configs_in_sample / vp_simulations, DIGIT_PRECISION) combinations = vp_analysis.get_variants_combinations(vp) variants = vp_analysis.get_variants(vp) combinations = variants # only for optional features analysis optional_choice = [True, False] for combi in combinations: # Real probabilities of each variant combination #selected_features = list(set(combi).union({vp})) #deselected_features = list(set(variants) - set(combi)) # only for optional features analysis original_combi = combi for choice in optional_choice: if choice: combi = [original_combi] selected_features = combi + [vp] deselected_features = [] else: combi = [] selected_features = [vp] deselected_features = [original_combi] combi_configurations = bdd.get_configurations(selected_features=selected_features, deselected_features=deselected_features) combi_configurations_ratio = round(len(combi_configurations) / len(vp_configurations), DIGIT_PRECISION) if len(combi_configurations) > 0: combi_defective_configs = [c for c in combi_configurations if jhipster_configurations[c]] combi_probability_defective_configs = round(len(combi_defective_configs) / len(combi_configurations), DIGIT_PRECISION) # Monte Carlo simulations simulations = math.ceil(len(combi_configurations) * PERCENTAGE_SIMULATIONS) # if simulations > len(combi_configurations): # simulations = len(combi_configurations) runs = RUNS defective_configs_in_sample_per_runs = [] for r in range(runs): sample_configurations = random.sample(combi_configurations, simulations) defective_configs_in_sample = [c for c in sample_configurations if jhipster_configurations[c]] defective_configs_in_sample_per_runs.append(len(defective_configs_in_sample)) median_defective_configs_in_sample = round(statistics.median(defective_configs_in_sample_per_runs), DIGIT_PRECISION) mean_defective_configs_in_sample = round(statistics.mean(defective_configs_in_sample_per_runs), DIGIT_PRECISION) std_defective_configs_in_sample = round(statistics.stdev(defective_configs_in_sample_per_runs), DIGIT_PRECISION) sample_median_probability_defective_configs = round(median_defective_configs_in_sample / simulations, DIGIT_PRECISION) file.write(f'{len(vps)}, {vp.name}, {len(vp_configurations)}, {len(vp_defective_configs)}, {vp_probability_defective_configs}, ' \ + f'{vp_simulations}, {vp_median_defective_configs_in_sample}, {vp_mean_defective_configs_in_sample}, {vp_std_defective_configs_in_sample}, {vp_sample_median_probability_defective_configs}, ' \ + f'{len(combinations)}, "{[str(f) for f in combi]}", {len(combi)}, {len(combi_configurations)}, {combi_configurations_ratio}, {len(combi_defective_configs)}, {combi_probability_defective_configs}, ' \ + f'{simulations}, {runs}, {median_defective_configs_in_sample}, {mean_defective_configs_in_sample}, {std_defective_configs_in_sample}, {sample_median_probability_defective_configs}\n') else: file.write(f'{len(vps)}, {vp.name}, {len(vp_configurations)}, {len(vp_defective_configs)}, {vp_probability_defective_configs}, ' \ + f'{vp_simulations}, {vp_median_defective_configs_in_sample}, {vp_mean_defective_configs_in_sample}, {vp_std_defective_configs_in_sample}, {vp_sample_median_probability_defective_configs}, ' \ + f'{len(combinations)}, "{[str(f) for f in combi]}", {len(combi)}, {len(combi_configurations)}, {combi_configurations_ratio}, {0}, {0}, ' \ + f'{"-"}, {"-"}, {"-"}, {"-"}, {"-"}, {"-"}\n')
def main(algorithm, simulations: int, mcts_stats_its): print("Problem 1 (simulated): Finding defective configurations in the jHipster feature model.") print("--------------------------------------------------------------------------------------") print("Setting up the problem...") print("Creating output folders...") if not os.path.exists(OUTPUT_RESULTS_PATH): os.makedirs(OUTPUT_RESULTS_PATH) if not os.path.exists(HEATMAP_PATH): os.makedirs(HEATMAP_PATH) if not os.path.exists(STATS_PATH): os.makedirs(STATS_PATH) print(f"Loading feature model: {jhipster.FM_FILE} ...") fide_parser = FeatureIDEParser(jhipster.FM_FILE, no_read_constraints=True) fm = fide_parser.transform() print(f"Feature model loaded with {len(fm.get_features())} features, {len(fm.get_constraints())} constraints, {len(fm.get_relations())} relations.") # Read the feature model as CNF model with complex constraints cnf_reader = CNFReader(jhipster.CNF_FILE) cnf_model = cnf_reader.transform() # AAFMs aafms_helper = AAFMsHelper(fm, cnf_model) print(f"Creating set of actions...") actions = TreeActionsList(fm) print(f"{actions.get_nof_actions()} actions.") problem_data = ProblemData(fm, aafms_helper, actions) # Read the jhipster configurations as a dict of FMConfiguration -> bool (failure) jhipster_configurations = jhipster.read_jHipster_feature_model_configurations() problem_data.jhipster_configurations = jhipster_configurations problem_data.sample = defaultdict(bool) print(f"Creating initial state (configuration)...") initial_config = FMConfiguration() initial_state = FailureConfigurationState(configuration=initial_config, data=problem_data) print(f"Initial state: {initial_state}") print("Problem setted up.") print(f"Running algorithm {str(algorithm)}...") # Stats mcts_stats = MCTSStats() n = 0 total_evaluations = 0 state = initial_state total_time_start = time.time() while state.reward() <= 0 and state.get_actions(): print(f"Input state {n}: {str(state)} -> valid={state.is_valid_configuration}, R={state.reward()}") time_start = time.time() new_state = algorithm.run(state) time_end = time.time() if isinstance(algorithm, MonteCarloTreeSearch): # Heat map (only for MCTS) heatmap = Heatmap(fm, algorithm.tree, algorithm.Q, algorithm.N, state) heatmap.extract_feature_knowledge() heatmap.serialize(HEATMAP_PATH + jhipster.FM_FILENAME + "-step" + str(n) + ".csv") else: algorithm.tree = {} # Stats mcts_stats.add_step(n, algorithm.tree, state, new_state, simulations, algorithm.n_evaluations, algorithm.n_positive_evaluations, time_end-time_start) total_evaluations += algorithm.n_evaluations algorithm.n_evaluations = 0 state = new_state n += 1 total_time_end = time.time() print("Algorithm finished.") print(f"Final state {n}: {str(state)} -> valid={state.is_valid_configuration}, R={state.reward()}") # Stats print("Serializing results...") mcts_stats.serialize(STATS_PATH + jhipster.FM_FILENAME + '-steps.csv') mcts_stats_its.add_step(str(algorithm), n, algorithm.tree, simulations, total_evaluations, algorithm.n_positive_evaluations, total_time_end-total_time_start) #mcts_stats_its.serialize(STATS_PATH + jhipster.FM_FILENAME + '-summary.csv') print("Done!")
def main(algorithm, simulations: int, input_fm: str, input_cnf_model: str=None): print("Problem: Reverse engineering of feature models.") print("-----------------------------------------------") base = os.path.basename(input_fm) input_fm_name = os.path.splitext(base)[0] print("Setting up the problem...") print("Creating output folders...") if not os.path.exists(HEATMAP_PATH): os.makedirs(HEATMAP_PATH) if not os.path.exists(STATS_PATH): os.makedirs(STATS_PATH) if not os.path.exists(GENERATED_FMS_PATH): os.makedirs(GENERATED_FMS_PATH) print(f"Loading feature model: {input_fm_name} ...") fide_parser = FeatureIDEParser(input_fm, no_read_constraints=(input_cnf_model is not None)) fm = fide_parser.transform() print(f"Feature model loaded with {len(fm.get_features())} features, {len(fm.get_constraints())} constraints, {len(fm.get_relations())} relations.") if input_cnf_model is not None: # Read the feature model as CNF model with complex constraints cnf_reader = CNFReader(input_cnf_model) cnf_model = cnf_reader.transform() else: cnf_model = None # Get configurations print("Generating configurations of the feature model...") aafms_helper = AAFMsHelper(fm, cnf_model) configurations = aafms_helper.get_configurations() print(f"#Configurations: {len(configurations)}") print(f"Creating initial state (empty feature model)...") initial_state = FMState(FeatureModel(None), configurations) print("Problem setted up.") print(f"Running algorithm {str(algorithm)}...") # Stats mcts_stats_re = MCTSStatsRE(STATS_PATH + input_fm_name + "-ReverseEngineering.log") n = 0 state = initial_state total_time_start = time.time() while not state.is_terminal(): print(f"State {n}: {[str(f) for f in state.feature_model.get_features()]} -> {state.reward()}") start_time = time.time() new_state = algorithm.run(state) end_time = time.time() if isinstance(algorithm, MonteCarloTreeSearch): mcts_stats_re.add_step(n, algorithm.tree, algorithm.Q, algorithm.N, state, new_state, simulations, end_time-start_time) else: algorithm.tree = {} state = new_state n += 1 total_time_end = time.time() print("Algorithm finished.") print(f"Final State {n}: {[str(f) for f in state.feature_model.get_features()]} -> {state.reward()}") # Get configurations path = GENERATED_FMS_PATH + state.feature_model.root.name + "." + UVLWritter.get_destination_extension() print(f"Serializing generated feature model in UVL format in {path}") uvl_writter = UVLWritter(path=path, source_model=state.feature_model) uvl_writter.transform() # Get configurations print("Generating configurations of the extracted feature model...") aafms_helper = AAFMsHelper(state.feature_model) new_configurations = aafms_helper.get_configurations() print("Results:") print(f"#Features: {len(state.feature_model.get_features())} -> {[str(f) for f in state.feature_model.get_features()]}") print(f"#Configurations: {len(new_configurations)}") relaxed_value = reduce(lambda count, c: count + (aafms_helper.is_valid_configuration(c)), configurations, 0) deficit_value = reduce(lambda count, c: count + (c not in new_configurations), configurations, 0) surplus_value = reduce(lambda count, c: count + (c not in configurations), new_configurations, 0) print(f"Input configurations captured (Relaxed objective function): {relaxed_value}") print(f"Deficit of configurations: {deficit_value}") print(f"Irrelevant configurations: {surplus_value}") print(f"Mininal difference (MinDiff) objective function (deficit_value + surplus_value): {deficit_value} + {surplus_value} = {deficit_value+surplus_value}") print(f"Final objective function (Relaxed - MinDiff): {relaxed_value - (deficit_value+surplus_value)}") print(f"Execution time: {total_time_end-total_time_start}")
def main(algorithm, simulations: int, input_fm: str, input_cnf_model: str = None, initial_config_features: list[str] = [], minimum: bool = False): print("Problem: Completion of partial configurations.") print("----------------------------------------------") print("Setting up the problem...") print("Creating output folders...") if not os.path.exists(OUTPUT_RESULTS_PATH): os.makedirs(OUTPUT_RESULTS_PATH) if not os.path.exists(HEATMAP_PATH): os.makedirs(HEATMAP_PATH) if not os.path.exists(STATS_PATH): os.makedirs(STATS_PATH) base = os.path.basename(input_fm) input_fm_name = os.path.splitext(base)[0] print(f"Loading feature model: {input_fm} ...") fide_parser = FeatureIDEParser(input_fm, no_read_constraints=(input_cnf_model is not None)) fm = fide_parser.transform() print( f"Feature model loaded with {len(fm.get_features())} features, {len(fm.get_constraints())} constraints, {len(fm.get_relations())} relations." ) if input_cnf_model is not None: # Read the feature model as CNF model with complex constraints cnf_reader = CNFReader(input_cnf_model) cnf_model = cnf_reader.transform() else: cnf_model = None # AAFMs aafms_helper = AAFMsHelper(fm, cnf_model) print(f"Creating set of actions...") actions = TreeActionsList(fm) print(f"{actions.get_nof_actions()} actions.") problem_data = ProblemData(fm, aafms_helper, actions) print(f"Creating initial state (configuration)...") if initial_config_features: elements = {} for feature in [ fm.get_feature_by_name(f) for f in initial_config_features ]: elements[feature] = True for p in fm_utils.select_parent_features(feature): elements[p] = True initial_config = FMConfiguration(elements=elements) else: initial_config = FMConfiguration() if minimum: initial_state = ValidMinimumConfigurationState( configuration=initial_config, data=problem_data) else: initial_state = ValidConfigurationState(configuration=initial_config, data=problem_data) print(f"Initial state: {initial_state}") print("Problem setted up.") print(f"Running algorithm {str(algorithm)}...") # Stats mcts_stats = MCTSStats() mcts_stats_its = MCTSStatsIts() n = 0 total_evaluations = 0 state = initial_state total_time_start = time.time() while not state.is_terminal(): print( f"Input state {n}: {str(state)} -> valid={state.is_valid_configuration}, R={state.reward()}" ) time_start = time.time() new_state = algorithm.run(state) time_end = time.time() if isinstance(algorithm, MonteCarloTreeSearch): # Heat map (only for MCTS) heatmap = Heatmap(fm, algorithm.tree, algorithm.Q, algorithm.N, state) heatmap.extract_feature_knowledge() heatmap.serialize(HEATMAP_PATH + input_fm_name + "-step" + str(n) + ".csv") else: algorithm.tree = {} # Stats mcts_stats.add_step(n, algorithm.tree, state, new_state, simulations, algorithm.n_evaluations, algorithm.n_positive_evaluations, time_end - time_start) total_evaluations += algorithm.n_evaluations algorithm.n_evaluations = 0 state = new_state n += 1 total_time_end = time.time() print("Algorithm finished.") print( f"Final state {n}: {str(state)} -> valid={state.is_valid_configuration}, R={state.reward()}" ) # Stats print("Serializing results...") mcts_stats.serialize(STATS_PATH + input_fm_name + '-steps.csv') mcts_stats_its.add_step(str(algorithm), n, algorithm.tree, simulations, total_evaluations, algorithm.n_positive_evaluations, total_time_end - total_time_start) mcts_stats_its.serialize(STATS_PATH + input_fm_name + '-summary.csv') # Heat for the whole process (not sure about its utility) if isinstance(algorithm, MonteCarloTreeSearch): heatmap = HeatmapFull(fm, algorithm.tree, algorithm.Q, algorithm.N) heatmap.extract_feature_knowledge() heatmap.serialize(HEATMAP_PATH + input_fm_name + "-full.csv") print("Done!")