def _evaluate_train_test_split_mod(scenario: ASlibScenario, approach, metrics, fold: int, on_training): test_scenario, train_scenario = scenario.get_split(indx=fold) if on_training: test_scenario = train_scenario approach_metric_values = np.zeros(len(metrics)) num_counted_test_values = 0 feature_data = test_scenario.feature_data.to_numpy() performance_data = test_scenario.performance_data.to_numpy() feature_cost_data = test_scenario.feature_cost_data.to_numpy( ) if test_scenario.feature_cost_data is not None else None for instance_id in range(0, len(test_scenario.instances)): X_test = feature_data[instance_id] y_test = performance_data[instance_id] accumulated_feature_time = 0 if test_scenario.feature_cost_data is not None and approach.get_name( ) != 'sbs' and approach.get_name() != 'oracle': feature_time = feature_cost_data[instance_id] accumulated_feature_time = np.sum(feature_time) contains_non_censored_value = False for y_element in y_test: if y_element < test_scenario.algorithm_cutoff_time: contains_non_censored_value = True if contains_non_censored_value: num_counted_test_values += 1 predicted_scores = approach.predict(X_test, instance_id) for i, metric in enumerate(metrics): runtime = metric.evaluate(y_test, predicted_scores, accumulated_feature_time, scenario.algorithm_cutoff_time) approach_metric_values[i] = (approach_metric_values[i] + runtime) approach_metric_values = np.true_divide(approach_metric_values, num_counted_test_values) print('PAR10: {0:.10f}'.format(approach_metric_values[0])) return approach_metric_values
def get_par10(self, scenario: ASlibScenario, fold: int): metrics = list() metrics.append(Par10Metric()) test_scenario, train_scenario = scenario.get_split(indx=fold) approach_metric_values = np.zeros(len(metrics)) num_counted_test_values = 0 feature_data = train_scenario.feature_data.to_numpy() performance_data = train_scenario.performance_data.to_numpy() feature_cost_data = train_scenario.feature_cost_data.to_numpy( ) if train_scenario.feature_cost_data is not None else None for instance_id in range(0, len(train_scenario.instances)): X_test = feature_data[instance_id] y_test = performance_data[instance_id] accumulated_feature_time = 0 if train_scenario.feature_cost_data is not None and self.get_name( ) != 'sbs' and self.get_name() != 'oracle': feature_time = feature_cost_data[instance_id] accumulated_feature_time = np.sum(feature_time) contains_non_censored_value = False for y_element in y_test: if y_element < train_scenario.algorithm_cutoff_time: contains_non_censored_value = True if contains_non_censored_value: num_counted_test_values += 1 predicted_scores = self.predict(X_test, instance_id, opt=True) for i, metric in enumerate(metrics): runtime = metric.evaluate(y_test, predicted_scores, accumulated_feature_time, scenario.algorithm_cutoff_time) approach_metric_values[i] = (approach_metric_values[i] + runtime) approach_metric_values = np.true_divide(approach_metric_values, num_counted_test_values) return approach_metric_values
def run_fold(self, config: Configuration, scenario:ASlibScenario, fold:int): ''' run a given fold of cross validation Arguments --------- scenario: aslib_scenario.aslib_scenario.ASlibScenario aslib scenario at hand config: Configuration parameter configuration to use for preprocessing fold: int fold id Returns ------- Stats() ''' self.logger.info("CV-Iteration: %d" % (fold)) test_scenario, training_scenario = scenario.get_split(indx=fold) feature_pre_pipeline, pre_solver, selector = self.fit( scenario=training_scenario, config=config) schedules = self.predict( test_scenario, config, feature_pre_pipeline, pre_solver, selector) val = Validator() if scenario.performance_type[0] == "runtime": stats = val.validate_runtime( schedules=schedules, test_scenario=test_scenario) elif scenario.performance_type[0] == "solution_quality": stats = val.validate_quality( schedules=schedules, test_scenario=test_scenario) else: raise ValueError("Unknown: %s" %(performance_type[0])) return stats
def _outer_cv(solver_fold, args, config): solver, fold = solver_fold # there are problems serializing the aslib scenario, so just read it again scenario = ASlibScenario() scenario.read_scenario(args.scenario) msg = "Solver: {}, Fold: {}".format(solver, fold) logger.info(msg) msg = "Constructing template pipeline" logger.info(msg) pipeline = _get_pipeline(args, config, scenario) msg = "Extracting solver and fold performance data" logger.info(msg) testing, training = scenario.get_split(fold) X_train = training.feature_data y_train = training.performance_data[solver].values if 'log_performance_data' in config: y_train = np.log1p(y_train) msg = "Fitting the pipeline" logger.info(msg) pipeline = pipeline.fit(X_train, y_train) out = string.Template(args.out) out = out.substitute(solver=solver, fold=fold) msg = "Writing fit pipeline to disk: {}".format(out) logger.info(msg) joblib.dump(pipeline, out) return pipeline
def run_fold(self, config: Configuration, scenario:ASlibScenario, fold:int, test_scenario=None, return_fit:bool=False): ''' run a given fold of cross validation Arguments --------- scenario: aslib_scenario.aslib_scenario.ASlibScenario aslib scenario at hand config: Configuration parameter configuration to use for preprocessing fold: int fold id test_scenario:aslib_scenario.aslib_scenario.ASlibScenario aslib scenario with test data for validation generated from <scenario> if None return_fit: bool optionally, the learned preprocessing options, presolver and selector can be returned Returns ------- Stats() (pre_pipeline, pre_solver, selector): only present if return_fit is True the pipeline components fit with the configuration options schedule: dict of string -> list of (solver, cutoff) pairs only present if return_fit is True the solver choices for each instance ''' if test_scenario is None: self.logger.info("CV-Iteration: %d" % (fold)) test_scenario, training_scenario = scenario.get_split(indx=fold) else: self.logger.info("Validation on test data") training_scenario = scenario feature_pre_pipeline, pre_solver, selector = self.fit( scenario=training_scenario, config=config) schedules = self.predict( test_scenario, config, feature_pre_pipeline, pre_solver, selector) val = Validator() if scenario.performance_type[0] == "runtime": stats = val.validate_runtime( schedules=schedules, test_scenario=test_scenario, train_scenario=training_scenario) elif scenario.performance_type[0] == "solution_quality": stats = val.validate_quality( schedules=schedules, test_scenario=test_scenario, train_scenario=training_scenario) else: raise ValueError("Unknown: %s" %(scenario.performance_type[0])) if return_fit: return stats, (feature_pre_pipeline, pre_solver, selector), schedules else: return stats
def _outer_cv(self, scenario: ASlibScenario, autofolio_config:dict=None, outer_cv_fold:int=None, out_template:str=None, smac_seed:int=42): ''' Evaluate on a scenario using an "outer" cross-fold validation scheme. In particular, this ensures that SMAC does not use the test set during hyperparameter optimization. Arguments --------- scenario: ASlibScenario ASlib Scenario at hand autofolio_config: dict, or None An optional dictionary of configuration options outer_cv_fold: int, or None If given, then only the single outer-cv fold is processed out_template: str, or None If given, the learned configurations are written to the specified locations. The string is considered a template, and "%fold%" will be replaced with the fold. smac_seed:int random seed for SMAC Returns ------- stats: validate.Stats Performance over all outer-cv folds ''' import string outer_stats = None # For each outer split outer_cv_folds = range(1, 11) if outer_cv_fold is not None: outer_cv_folds = range(outer_cv_fold, outer_cv_fold+1) for cv_fold in outer_cv_folds: # Use ‘ASlibScenario.get_split()’ to get the outer split outer_testing, outer_training = scenario.get_split(cv_fold) msg = ">>>>> Outer CV fold: {} <<<<<".format(cv_fold) self.logger.info(msg) # Use ASlibScenario.create_cv_splits() to get an inner-cv outer_training.create_cv_splits(n_folds=10) # Use ‘AutoFolio.get_tuned_config()’ to tune on inner-cv config = self.get_tuned_config( outer_training, autofolio_config=autofolio_config, seed=smac_seed ) # Use `AutoFolio.run_fold()’ to get the performance on the outer split stats, fit, schedule = self.run_fold( config, scenario, cv_fold, return_fit=True ) feature_pre_pipeline, pre_solver, selector = fit if outer_stats is None: outer_stats = stats else: outer_stats.merge(stats) # save the model, if given an output location if out_template is not None: out_template_ = string.Template(out_template) model_fn = out_template_.substitute(fold=cv_fold, type="pkl") msg = "Writing model to: {}".format(model_fn) self.logger.info(msg) self._save_model( model_fn, scenario, feature_pre_pipeline, pre_solver, selector, config ) # convert the schedule to a data frame schedule_df = pd.Series(schedule, name="solver") schedule_df.index.name = "instance" schedule_df = schedule_df.reset_index() # just keep the solver name; we don't care about the time # x[0] gets the first pair in the schedule list # and x[0][0] gets the name of the solver from that pair schedule_df['solver'] = schedule_df['solver'].apply(lambda x: x[0][0]) selections_fn = out_template_.substitute(fold=cv_fold, type="csv") msg = "Writing solver choices to: {}".format(selections_fn) self.logger.info(msg) schedule_df.to_csv(selections_fn, index=False) self.logger.info(">>>>> Final Stats <<<<<") outer_stats.show()
def evaluate_train_test_split(scenario: ASlibScenario, approach, metrics, fold: int, amount_of_training_instances: int, train_status: str): test_scenario, train_scenario = scenario.get_split(indx=fold) if train_status != 'all': train_scenario = copy.deepcopy(train_scenario) threshold = train_scenario.algorithm_cutoff_time if train_status == 'clip_censored': train_scenario.performance_data = train_scenario.performance_data.clip( upper=threshold) elif train_status == 'ignore_censored': train_scenario.performance_data = train_scenario.performance_data.replace( 10 * threshold, np.nan) if approach.get_name() == 'oracle' or approach.get_name( ) == 'virtual_sbs_with_feature_costs': approach.fit(test_scenario, fold, amount_of_training_instances) else: approach.fit(train_scenario, fold, amount_of_training_instances) approach_metric_values = np.zeros(len(metrics)) num_counted_test_values = 0 feature_data = test_scenario.feature_data.to_numpy() performance_data = test_scenario.performance_data.to_numpy() feature_cost_data = test_scenario.feature_cost_data.to_numpy( ) if test_scenario.feature_cost_data is not None else None instancewise_result_strings = list() simple_runtime_metric = RuntimeMetric() for instance_id in range(0, len(test_scenario.instances)): X_test = feature_data[instance_id] y_test = performance_data[instance_id] # compute feature time accumulated_feature_time = 0 if test_scenario.feature_cost_data is not None and approach.get_name( ) != 'sbs' and approach.get_name() != 'oracle': feature_time = feature_cost_data[instance_id] accumulated_feature_time = np.sum(feature_time) #compute the values of the different metrics predicted_scores = approach.predict(X_test, instance_id) num_counted_test_values += 1 for i, metric in enumerate(metrics): runtime = metric.evaluate(y_test, predicted_scores, accumulated_feature_time, scenario.algorithm_cutoff_time) approach_metric_values[i] = (approach_metric_values[i] + runtime) # store runtimes on a per instance basis in ASLib format runtime = simple_runtime_metric.evaluate( y_test, predicted_scores, accumulated_feature_time, scenario.algorithm_cutoff_time) run_status_to_print = "ok" if runtime < scenario.algorithm_cutoff_time else "timeout" line_to_store = test_scenario.instances[ instance_id] + ",1," + approach.get_name() + "," + str( runtime) + "," + run_status_to_print instancewise_result_strings.append(line_to_store) write_instance_wise_results_to_file(instancewise_result_strings, scenario.scenario) approach_metric_values = np.true_divide(approach_metric_values, num_counted_test_values) for i, metric in enumerate(metrics): print(metrics[i].get_name() + ': {0:.10f}'.format(approach_metric_values[i])) return approach_metric_values
def main(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="Test models learned with train-as-auto-sklearn. It " "writes the predictions to disk as a \"long\" data frame. The output " "file is in gzipped csv format.") parser.add_argument('scenario', help="The ASlib scenario") parser.add_argument('model_template', help="A template string for the filenames for " "the learned models. ${solver} and ${fold} are the template part of " "the string. It is probably necessary to surround this argument with " "single quotes in order to prevent shell replacement of the template " "parts.") parser.add_argument('out', help="The output csv file") parser.add_argument('--config', help="A (yaml) config file which " "specifies options controlling the learner behavior") logging_utils.add_logging_options(parser) args = parser.parse_args() logging_utils.update_logging(args) msg = "Loading ASlib scenario" logger.info(msg) scenario = ASlibScenario() scenario.read_scenario(args.scenario) if args.config is not None: msg = "Loading yaml config file" logger.info(msg) config = yaml.load(open(args.config)) else: config = {} msg = "Creating string templates" logger.info(msg) model_template = string.Template(args.model_template) msg = "Finding folds from ASlib scenario" logger.info(msg) folds = [int(i) for i in scenario.cv_data['fold'].unique()] folds = sorted(folds) msg = "Making predictions" logger.info(msg) all_predictions = [] it = itertools.product(scenario.algorithms, folds) for solver, fold in it: model_file = model_template.substitute(solver=solver, fold=fold) if not os.path.exists(model_file): msg = "Could not find model file. Skipping: {}".format(model_file) logger.warning(msg) continue try: model = joblib.load(model_file) except: msg = ("Problem loading the model file. Skipping: {}".format( model_file)) logger.warning(msg) continue msg = "Processing. solver: {}. fold: {}".format(solver, fold) logger.info(msg) testing, training = scenario.get_split(fold) y_pred = model.predict(testing.feature_data) if 'log_performance_data': # exp transform it back out y_pred = np.expm1(y_pred) pred_df = pd.DataFrame() pred_df['instance_id'] = testing.feature_data.index pred_df['solver'] = solver pred_df['fold'] = fold pred_df['actual'] = testing.performance_data[solver].values pred_df['predicted'] = y_pred all_predictions.append(pred_df) msg = "Joining all predictions in a long data frame" logger.info(msg) all_predictions = pd.concat(all_predictions) msg = "Writing predictions to disk" logger.info(msg) utils.write_df(all_predictions, args.out, index=False)
def evaluate_train_test_split(scenario: ASlibScenario, approach, metrics, fold: int, amount_of_training_instances: int, train_status: str): test_scenario, train_scenario = scenario.get_split(indx=fold) if train_status != 'all': train_scenario = copy.deepcopy(train_scenario) threshold = train_scenario.algorithm_cutoff_time if train_status == 'clip_censored': train_scenario.performance_data = train_scenario.performance_data.clip( upper=threshold) elif train_status == 'ignore_censored': train_scenario.performance_data = train_scenario.performance_data.replace( 10 * threshold, np.nan) if approach.get_name() == 'oracle': approach.fit(test_scenario, fold, amount_of_training_instances) else: approach.fit(train_scenario, fold, amount_of_training_instances) approach_metric_values = np.zeros(len(metrics)) num_counted_test_values = 0 feature_data = test_scenario.feature_data.to_numpy() performance_data = test_scenario.performance_data.to_numpy() feature_cost_data = test_scenario.feature_cost_data.to_numpy( ) if test_scenario.feature_cost_data is not None else None for instance_id in range(0, len(test_scenario.instances)): #logger.debug("Test instance " + str(instance_id) + "/" + str(len(test_scenario.instances))) X_test = feature_data[instance_id] y_test = performance_data[instance_id] accumulated_feature_time = 0 if test_scenario.feature_cost_data is not None and approach.get_name( ) != 'sbs' and approach.get_name() != 'oracle': feature_time = feature_cost_data[instance_id] accumulated_feature_time = np.sum(feature_time) contains_non_censored_value = False for y_element in y_test: if y_element < test_scenario.algorithm_cutoff_time: contains_non_censored_value = True if contains_non_censored_value: num_counted_test_values += 1 predicted_scores = approach.predict(X_test, instance_id) for i, metric in enumerate(metrics): runtime = metric.evaluate(y_test, predicted_scores, accumulated_feature_time, scenario.algorithm_cutoff_time) approach_metric_values[i] = (approach_metric_values[i] + runtime) approach_metric_values = np.true_divide(approach_metric_values, num_counted_test_values) print('PAR10: {0:.10f}'.format(approach_metric_values[0])) return approach_metric_values
class ASAPy(object): def __init__(self, output_dn: str = ".", plot_log_perf: bool = False): ''' Constructor Arguments --------- output_dn:str output directory name ''' self.logger = logging.getLogger("ASAPy") self.scenario = None self.output_dn = output_dn self.plot_log_perf = plot_log_perf if not os.path.isdir(self.output_dn): os.mkdir(self.output_dn) def read_scenario_ASlib(self, scenario_dn: str): ''' Read scenario from ASlib format Arguments --------- scenario_dn: str Scenario directory name ''' self.scenario = ASlibScenario() self.scenario.read_scenario(dn=scenario_dn) def read_scenario_CSV(self, csv_data: namedtuple): ''' Read scenario from ASlib format Arguments --------- csv_data: namedtuple namedtuple with the following fields: "perf_csv", "feat_csv", "obj", "cutoff", "maximize", "cv_csv" "cv_csv" can be None ''' self.scenario = ASlibScenario() self.scenario.read_from_csv(perf_fn=csv_data.perf_csv, feat_fn=csv_data.feat_csv, objective=csv_data.obj, runtime_cutoff=csv_data.cutoff, maximize=csv_data.maximize, cv_fn=csv_data.cv_csv) def get_default_config(self): ''' get default configuration which enables all plots Returns ------- dict ''' config = { "Performance Analysis": { "Status bar plot": True, "Box plot": True, "Violin plot": True, "CDF plot": True, "Scatter plots": True, "Correlation plot": True, "Contribution of algorithms": True, "Critical Distance Diagram": True, "Footprints": True, "Instance Hardness": True, "Baselines": True }, "Feature Analysis": { "Status Bar Plot": True, "Violin and box plots": True, "Correlation plot": True, "Feature importance": True, "Clustering": True, "CDF plot on feature costs": True } } return config def print_config(self): ''' generate template for config file ''' print(json.dumps(self.get_default_config(), indent=2)) def load_config(self, fn: str): ''' load config from file Arguments --------- fn: str file name with config in json format Returns ------- config: dict ''' with open(fn) as fp: config = json.load(fp) return config def main(self, config: dict, max_algos: int = 20, only_fold: int = None): ''' main method Arguments --------- config: dict configuration that enables or disables plots max_algos: int maximum number of algos to consider; if more are available, we take the n best algorithm on average performance only_fold: int use only the given <only_fold> cv-fold data for analyze ''' if only_fold: self.logger.info("Using only test data from %d cv-split" % (only_fold)) _, self.scenario = self.scenario.get_split(only_fold) n_prev_algos = None if self.scenario is None: raise ValueError( "Please first read in Scenario data; use scenario input or csv input" ) if self.scenario.performance_type[ 0] == "solution_quality" and self.scenario.maximize[0]: # revoke inverting the performance as done in the scenario reader self.scenario.performance_data *= -1 self.logger.info("Revoke * -1 on performance data") if len(self.scenario.algorithms) > max_algos: n_prev_algos = len(self.scenario.algorithms) self.logger.warning( "We reduce the algorithms to the greedy selected VBS-improving algorithms (at most %d)" % (max_algos)) pa = PerformanceAnalysis(output_dn=self.output_dn, scenario=self.scenario) algos_score = pa.reduce_algos(max_algos=max_algos) best_algos = [a[0] for a in algos_score] self.scenario.algorithms = best_algos self.scenario.performance_data = self.scenario.performance_data[ best_algos] self.scenario.runstatus_data = self.scenario.runstatus_data[ best_algos] data = OrderedDict() # meta data meta_data_df = self.get_meta_data() data["Meta Data"] = {"table": meta_data_df.to_html(header=False)} #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # performance analysis #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> if config.get("Performance Analysis"): pa = PerformanceAnalysis(output_dn=self.output_dn, scenario=self.scenario) data["Performance Analysis"] = OrderedDict() if n_prev_algos is not None: data["Performance Analysis"][ "tooltip"] = "To provide a clear overview, we reduced the number of algorithms (%d) to the greedily selected most VBS-improving %d algorithms." % ( n_prev_algos, len(self.scenario.algorithms)) if config["Performance Analysis"].get("Baselines"): baseline_table = pa.get_baselines() data["Performance Analysis"]["Baselines"] = { "tooltip": "Baslines: Best Single Algorithm (i.e., best algorithm on average across instances), Virtual Best Solver (aka Oracle, i.e., average across best performance per instance; theoretical best algorithm selector)", "table": baseline_table } if config["Performance Analysis"].get("Status bar plot"): status_plot = pa.get_bar_status_plot() data["Performance Analysis"]["Status bar plot"] = { "tooltip": "Stacked bar plots for runstatus of each algorithm", "figure": status_plot } # get box plot if config["Performance Analysis"].get("Box plot"): box_plot = pa.get_box_plots(plot_log_perf=self.plot_log_perf) data["Performance Analysis"]["Box plot"] = { "tooltip": "Box plots to show the performance distribution of each algorithm", "figure": box_plot } # get violin plot if config["Performance Analysis"].get("Violin plot"): violion_plot = pa.get_violin_plots( plot_log_perf=self.plot_log_perf) data["Performance Analysis"]["Violin plot"] = { "tooltip": "Violin plots to show the probablity density of each algorithm's performance. Also showing the median (middle line) and min/max value.", "figure": violion_plot } # get cdf plot if config["Performance Analysis"].get("CDF plot"): cdf_plot = pa.get_cdf_plots(plot_log_perf=self.plot_log_perf) data["Performance Analysis"]["CDF plot"] = { "tooltip": "Cumulative Distribution function (CDF) plots. At each point x (e.g., running time cutoff), how many of the instances (in percentage) can be solved. Better algorithms have a higher curve for minimization problems.", "figure": cdf_plot } # get cd diagram if config["Performance Analysis"].get("Critical Distance Diagram"): cd_plot = pa.get_cd_diagram() data["Performance Analysis"]["Critical Distance Diagram"] = { "tooltip": "Critical Distance (CD) diagram based on a Nemenyi two tailed test using average rankings. CD (top left) shows the critical distance. Distances larger than CD corresponds to a statistical significant difference in the ranking. We show only the best 20 ranked algorithms.", "figure": cd_plot } # generate scatter plots if config["Performance Analysis"].get("Scatter plots"): scatter_plots = pa.scatter_plots( plot_log_perf=self.plot_log_perf) data["Performance Analysis"]["Scatter plots"] = OrderedDict({ "tooltip": "Scatter plot to compare the performance of two algorithms on all instances -- each dot represents one instance." }) scatter_plots = sorted(scatter_plots, key=lambda x: x[0] + x[1]) for plot_tuple in scatter_plots: key = "%s vs %s" % (plot_tuple[0], plot_tuple[1]) data["Performance Analysis"]["Scatter plots"][key] = { "figure": plot_tuple[2] } # generate correlation plot if config["Performance Analysis"].get("Correlation plot"): correlation_plot = pa.correlation_plot() data["Performance Analysis"]["Correlation plot"] = { "tooltip": "Correlation based on Spearman Correlation Coefficient between all algorithms and clustered with Wards hierarchical clustering approach. Darker fields corresponds to a larger correlation between the algorithms. See [Xu et al SAT 2012]", "figure": correlation_plot } # get contribution values if config["Performance Analysis"].get( "Contribution of algorithms"): avg_fn, marg_fn, shap_fn = pa.get_contribution_values() data["Performance Analysis"][ "Contribution of algorithms"] = OrderedDict( {"tooltip": "Contribution of each algorithm"}) data["Performance Analysis"]["Contribution of algorithms"][ "Average Performance"] = { "figure": avg_fn } data["Performance Analysis"]["Contribution of algorithms"][ "Marginal Contribution"] = { "figure": marg_fn, "tooltip": "Marginal contribution to the virtual best solver (VBS, aka oracle) (i.e., how much decreases the VBS performance by removing the algorithm; higher value correspond to more importance). See [Xu et al SAT 2012]" } data["Performance Analysis"]["Contribution of algorithms"][ "Shapley Values"] = { "figure": shap_fn, "tooltip": "Shapley values (i.e., marginal contribution across all possible subsets of portfolios; again higher values corresponds to more importance; see [Frechette et al AAAI'16]. For running time scenarios, the metric is cutoff - running time; for non-running time, the metric is (worst performance across all instances and algorithms)- performance." } portfolio_table = pa.get_greedy_portfolio_constr() data["Performance Analysis"]["Contribution of algorithms"][ "Portfolio Construction"] = { "tooltip": "Starting with the Single Best Solver, iteratively add algorithms to the portfolio by optimizing VBS", "table": portfolio_table } # generate footprint plots if config["Performance Analysis"].get("Footprints"): footprints_plots = pa.get_footprints() data["Performance Analysis"]["Footprints"] = OrderedDict({ "tooltip": "Footprints of algorithms (instances red marked if the algorithm is at most 5% away from oracle performance) in 2-d PCA feature space. Inspired by [Smith-Miles et al. Computers & OR 2014]" }) footprints_plots = sorted(footprints_plots, key=lambda x: x[0]) for plot_tuple in footprints_plots: key = "%s" % (plot_tuple[0]) data["Performance Analysis"]["Footprints"][key] = { "html": plot_tuple[1], "figure": plot_tuple[2] } # generate instance hardness plot if config["Performance Analysis"].get("Instance Hardness"): hardness_plot = pa.instance_hardness() data["Performance Analysis"]["Instance Hardness"] = { "tooltip": "Projecting instances into 2d PCA feature space; the color encodes the number of algorithms that perform within 5% of the oracle performance.", "figure": hardness_plot[1], "html": hardness_plot[0] } #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # feature analysis #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> if config.get("Feature Analysis"): data["Feature Analysis"] = OrderedDict() fa = FeatureAnalysis(output_dn=self.output_dn, scenario=self.scenario) if config["Feature Analysis"].get("Status Bar Plot"): status_plot = fa.get_bar_status_plot() data["Feature Analysis"]["Status Bar Plot"] = { "tooltip": "Stacked bar plots for runstatus of each feature groupe", "figure": status_plot } # box and violin plots if config["Feature Analysis"].get("Violin and box plots"): name_plots = fa.get_box_violin_plots() data["Feature Analysis"]["Violin and box plots"] = OrderedDict({ "tooltip": "Violin and Box plots to show the distribution of each instance feature. We removed NaN from the data." }) for plot_tuple in name_plots: key = "%s" % (plot_tuple[0]) data["Feature Analysis"]["Violin and box plots"][key] = { "figure": plot_tuple[1] } # correlation plot if config["Feature Analysis"].get("Correlation plot"): correlation_plot = fa.correlation_plot() data["Feature Analysis"]["Correlation plot"] = { "tooltip": "Correlation based on Pearson product-moment correlation coefficients between all features and clustered with Wards hierarchical clustering approach. Darker fields corresponds to a larger correlation between the features.", "figure": correlation_plot } # feature importance if config["Feature Analysis"].get("Feature importance"): importance_plot = fa.feature_importance() data["Feature Analysis"]["Feature importance"] = { "tooltip": "Using the approach of SATZilla'11, we train a cost-sensitive random forest for each pair of algorithms and average the feature importance (using gini as splitting criterion) across all forests. We show the median, 25th and 75th percentiles across all random forests of the 15 most important features.", "figure": importance_plot } # cluster instances in feature space if config["Feature Analysis"].get("Clustering"): cluster_plot = fa.cluster_instances() data["Feature Analysis"]["Clustering"] = { "tooltip": "Clustering instances in 2d; the color encodes the cluster assigned to each cluster. Similar to ISAC, we use a k-means to cluster the instances in the feature space. As pre-processing, we use standard scaling and a PCA to 2 dimensions. To guess the number of clusters, we use the silhouette score on the range of 2 to 12 in the number of clusters", "figure": cluster_plot } # get cdf plot if self.scenario.feature_cost_data is not None and config[ "Feature Analysis"].get("CDF plot on feature costs"): cdf_plot = fa.get_feature_cost_cdf_plot() data["Feature Analysis"]["CDF plot on feature costs"] = { "tooltip": "Cumulative Distribution function (CDF) plots. At each point x (e.g., running time cutoff), for how many of the instances (in percentage) have we computed the instance features. Faster feature computation steps have a higher curve. Missing values are imputed with the maximal value (or running time cutoff).", "figure": cdf_plot } self.create_html(data=data) def create_html(self, data: OrderedDict): ''' create html report ''' html_builder = HTMLBuilder(output_dn=self.output_dn, scenario_name=self.scenario.scenario) html_builder.generate_html(data) def get_meta_data(self): ''' read meta data from self.scenario and generate a pandas.Dataframe with it ''' data = [] data.append(("Number of instances", len(self.scenario.instances))) data.append(("Number of algorithms", len(self.scenario.algorithms))) data.append( ("Performance measure", self.scenario.performance_measure[0])) data.append(("Performance type", self.scenario.performance_type[0])) data.append(("Maximize?", str(self.scenario.maximize[0]))) if self.scenario.algorithm_cutoff_time: data.append(("Running time cutoff (algorithm)", str(self.scenario.algorithm_cutoff_time))) if self.scenario.algorithm_cutoff_memory: data.append(("Memory cutoff (algorithm)", str(self.scenario.algorithm_cutoff_memory))) if self.scenario.features_cutoff_time: data.append(("Running time cutoff (features)", str(self.scenario.features_cutoff_time))) if self.scenario.features_cutoff_memory: data.append(("Memory cutoff (Features)", str(self.scenario.features_cutoff_memory))) data.append(("# Deterministic features", len(self.scenario.features_deterministic))) data.append( ("# Stochastic features", len(self.scenario.features_stochastic))) data.append(("# Feature groups", len(self.scenario.feature_steps))) data.append(("# Deterministic algorithms", len(self.scenario.algortihms_deterministics))) data.append(("# Stochastic algorithms", len(self.scenario.algorithms_stochastic))) if self.scenario.feature_cost_data is not None: data.append(("Feature costs provided?", "True")) else: data.append(("Feature costs provided?", "False")) meta_data = pd.DataFrame(data=list(map(lambda x: x[1], data)), index=list(map(lambda x: x[0], data)), columns=[""]) return meta_data