def _evaluate_train_test_split_mod(scenario: ASlibScenario, approach, metrics,
fold: int, on_training):
test_scenario, train_scenario = scenario.get_split(indx=fold)
if on_training:
test_scenario = train_scenario
approach_metric_values = np.zeros(len(metrics))
num_counted_test_values = 0
feature_data = test_scenario.feature_data.to_numpy()
performance_data = test_scenario.performance_data.to_numpy()
feature_cost_data = test_scenario.feature_cost_data.to_numpy(
) if test_scenario.feature_cost_data is not None else None
for instance_id in range(0, len(test_scenario.instances)):
X_test = feature_data[instance_id]
y_test = performance_data[instance_id]
accumulated_feature_time = 0
if test_scenario.feature_cost_data is not None and approach.get_name(
) != 'sbs' and approach.get_name() != 'oracle':
feature_time = feature_cost_data[instance_id]
accumulated_feature_time = np.sum(feature_time)
contains_non_censored_value = False
for y_element in y_test:
if y_element < test_scenario.algorithm_cutoff_time:
contains_non_censored_value = True
if contains_non_censored_value:
num_counted_test_values += 1
predicted_scores = approach.predict(X_test, instance_id)
for i, metric in enumerate(metrics):
runtime = metric.evaluate(y_test, predicted_scores,
accumulated_feature_time,
scenario.algorithm_cutoff_time)
approach_metric_values[i] = (approach_metric_values[i] +
runtime)
approach_metric_values = np.true_divide(approach_metric_values,
num_counted_test_values)
print('PAR10: {0:.10f}'.format(approach_metric_values[0]))
return approach_metric_values
def get_par10(self, scenario: ASlibScenario, fold: int):
metrics = list()
metrics.append(Par10Metric())
test_scenario, train_scenario = scenario.get_split(indx=fold)
approach_metric_values = np.zeros(len(metrics))
num_counted_test_values = 0
feature_data = train_scenario.feature_data.to_numpy()
performance_data = train_scenario.performance_data.to_numpy()
feature_cost_data = train_scenario.feature_cost_data.to_numpy(
) if train_scenario.feature_cost_data is not None else None
for instance_id in range(0, len(train_scenario.instances)):
X_test = feature_data[instance_id]
y_test = performance_data[instance_id]
accumulated_feature_time = 0
if train_scenario.feature_cost_data is not None and self.get_name(
) != 'sbs' and self.get_name() != 'oracle':
feature_time = feature_cost_data[instance_id]
accumulated_feature_time = np.sum(feature_time)
contains_non_censored_value = False
for y_element in y_test:
if y_element < train_scenario.algorithm_cutoff_time:
contains_non_censored_value = True
if contains_non_censored_value:
num_counted_test_values += 1
predicted_scores = self.predict(X_test, instance_id, opt=True)
for i, metric in enumerate(metrics):
runtime = metric.evaluate(y_test, predicted_scores,
accumulated_feature_time,
scenario.algorithm_cutoff_time)
approach_metric_values[i] = (approach_metric_values[i] +
runtime)
approach_metric_values = np.true_divide(approach_metric_values,
num_counted_test_values)
return approach_metric_values
def run_fold(self, config: Configuration, scenario:ASlibScenario, fold:int):
'''
run a given fold of cross validation
Arguments
---------
scenario: aslib_scenario.aslib_scenario.ASlibScenario
aslib scenario at hand
config: Configuration
parameter configuration to use for preprocessing
fold: int
fold id
Returns
-------
Stats()
'''
self.logger.info("CV-Iteration: %d" % (fold))
test_scenario, training_scenario = scenario.get_split(indx=fold)
feature_pre_pipeline, pre_solver, selector = self.fit(
scenario=training_scenario, config=config)
schedules = self.predict(
test_scenario, config, feature_pre_pipeline, pre_solver, selector)
val = Validator()
if scenario.performance_type[0] == "runtime":
stats = val.validate_runtime(
schedules=schedules, test_scenario=test_scenario)
elif scenario.performance_type[0] == "solution_quality":
stats = val.validate_quality(
schedules=schedules, test_scenario=test_scenario)
else:
raise ValueError("Unknown: %s" %(performance_type[0]))
return stats
def _outer_cv(solver_fold, args, config):
solver, fold = solver_fold
# there are problems serializing the aslib scenario, so just read it again
scenario = ASlibScenario()
scenario.read_scenario(args.scenario)
msg = "Solver: {}, Fold: {}".format(solver, fold)
logger.info(msg)
msg = "Constructing template pipeline"
logger.info(msg)
pipeline = _get_pipeline(args, config, scenario)
msg = "Extracting solver and fold performance data"
logger.info(msg)
testing, training = scenario.get_split(fold)
X_train = training.feature_data
y_train = training.performance_data[solver].values
if 'log_performance_data' in config:
y_train = np.log1p(y_train)
msg = "Fitting the pipeline"
logger.info(msg)
pipeline = pipeline.fit(X_train, y_train)
out = string.Template(args.out)
out = out.substitute(solver=solver, fold=fold)
msg = "Writing fit pipeline to disk: {}".format(out)
logger.info(msg)
joblib.dump(pipeline, out)
return pipeline
def run_fold(self, config: Configuration, scenario:ASlibScenario, fold:int, test_scenario=None, return_fit:bool=False):
'''
run a given fold of cross validation
Arguments
---------
scenario: aslib_scenario.aslib_scenario.ASlibScenario
aslib scenario at hand
config: Configuration
parameter configuration to use for preprocessing
fold: int
fold id
test_scenario:aslib_scenario.aslib_scenario.ASlibScenario
aslib scenario with test data for validation
generated from <scenario> if None
return_fit: bool
optionally, the learned preprocessing options, presolver and
selector can be returned
Returns
-------
Stats()
(pre_pipeline, pre_solver, selector):
only present if return_fit is True
the pipeline components fit with the configuration options
schedule: dict of string -> list of (solver, cutoff) pairs
only present if return_fit is True
the solver choices for each instance
'''
if test_scenario is None:
self.logger.info("CV-Iteration: %d" % (fold))
test_scenario, training_scenario = scenario.get_split(indx=fold)
else:
self.logger.info("Validation on test data")
training_scenario = scenario
feature_pre_pipeline, pre_solver, selector = self.fit(
scenario=training_scenario, config=config)
schedules = self.predict(
test_scenario, config, feature_pre_pipeline, pre_solver, selector)
val = Validator()
if scenario.performance_type[0] == "runtime":
stats = val.validate_runtime(
schedules=schedules, test_scenario=test_scenario, train_scenario=training_scenario)
elif scenario.performance_type[0] == "solution_quality":
stats = val.validate_quality(
schedules=schedules, test_scenario=test_scenario, train_scenario=training_scenario)
else:
raise ValueError("Unknown: %s" %(scenario.performance_type[0]))
if return_fit:
return stats, (feature_pre_pipeline, pre_solver, selector), schedules
else:
return stats
def _outer_cv(self, scenario: ASlibScenario, autofolio_config:dict=None,
outer_cv_fold:int=None, out_template:str=None,
smac_seed:int=42):
'''
Evaluate on a scenario using an "outer" cross-fold validation
scheme. In particular, this ensures that SMAC does not use the test
set during hyperparameter optimization.
Arguments
---------
scenario: ASlibScenario
ASlib Scenario at hand
autofolio_config: dict, or None
An optional dictionary of configuration options
outer_cv_fold: int, or None
If given, then only the single outer-cv fold is processed
out_template: str, or None
If given, the learned configurations are written to the
specified locations. The string is considered a template, and
"%fold%" will be replaced with the fold.
smac_seed:int
random seed for SMAC
Returns
-------
stats: validate.Stats
Performance over all outer-cv folds
'''
import string
outer_stats = None
# For each outer split
outer_cv_folds = range(1, 11)
if outer_cv_fold is not None:
outer_cv_folds = range(outer_cv_fold, outer_cv_fold+1)
for cv_fold in outer_cv_folds:
# Use ‘ASlibScenario.get_split()’ to get the outer split
outer_testing, outer_training = scenario.get_split(cv_fold)
msg = ">>>>> Outer CV fold: {} <<<<<".format(cv_fold)
self.logger.info(msg)
# Use ASlibScenario.create_cv_splits() to get an inner-cv
outer_training.create_cv_splits(n_folds=10)
# Use ‘AutoFolio.get_tuned_config()’ to tune on inner-cv
config = self.get_tuned_config(
outer_training,
autofolio_config=autofolio_config,
seed=smac_seed
)
# Use `AutoFolio.run_fold()’ to get the performance on the outer split
stats, fit, schedule = self.run_fold(
config,
scenario,
cv_fold,
return_fit=True
)
feature_pre_pipeline, pre_solver, selector = fit
if outer_stats is None:
outer_stats = stats
else:
outer_stats.merge(stats)
# save the model, if given an output location
if out_template is not None:
out_template_ = string.Template(out_template)
model_fn = out_template_.substitute(fold=cv_fold, type="pkl")
msg = "Writing model to: {}".format(model_fn)
self.logger.info(msg)
self._save_model(
model_fn,
scenario,
feature_pre_pipeline,
pre_solver,
selector,
config
)
# convert the schedule to a data frame
schedule_df = pd.Series(schedule, name="solver")
schedule_df.index.name = "instance"
schedule_df = schedule_df.reset_index()
# just keep the solver name; we don't care about the time
# x[0] gets the first pair in the schedule list
# and x[0][0] gets the name of the solver from that pair
schedule_df['solver'] = schedule_df['solver'].apply(lambda x: x[0][0])
selections_fn = out_template_.substitute(fold=cv_fold, type="csv")
msg = "Writing solver choices to: {}".format(selections_fn)
self.logger.info(msg)
schedule_df.to_csv(selections_fn, index=False)
self.logger.info(">>>>> Final Stats <<<<<")
outer_stats.show()
def evaluate_train_test_split(scenario: ASlibScenario, approach, metrics,
fold: int, amount_of_training_instances: int,
train_status: str):
test_scenario, train_scenario = scenario.get_split(indx=fold)
if train_status != 'all':
train_scenario = copy.deepcopy(train_scenario)
threshold = train_scenario.algorithm_cutoff_time
if train_status == 'clip_censored':
train_scenario.performance_data = train_scenario.performance_data.clip(
upper=threshold)
elif train_status == 'ignore_censored':
train_scenario.performance_data = train_scenario.performance_data.replace(
10 * threshold, np.nan)
if approach.get_name() == 'oracle' or approach.get_name(
) == 'virtual_sbs_with_feature_costs':
approach.fit(test_scenario, fold, amount_of_training_instances)
else:
approach.fit(train_scenario, fold, amount_of_training_instances)
approach_metric_values = np.zeros(len(metrics))
num_counted_test_values = 0
feature_data = test_scenario.feature_data.to_numpy()
performance_data = test_scenario.performance_data.to_numpy()
feature_cost_data = test_scenario.feature_cost_data.to_numpy(
) if test_scenario.feature_cost_data is not None else None
instancewise_result_strings = list()
simple_runtime_metric = RuntimeMetric()
for instance_id in range(0, len(test_scenario.instances)):
X_test = feature_data[instance_id]
y_test = performance_data[instance_id]
# compute feature time
accumulated_feature_time = 0
if test_scenario.feature_cost_data is not None and approach.get_name(
) != 'sbs' and approach.get_name() != 'oracle':
feature_time = feature_cost_data[instance_id]
accumulated_feature_time = np.sum(feature_time)
#compute the values of the different metrics
predicted_scores = approach.predict(X_test, instance_id)
num_counted_test_values += 1
for i, metric in enumerate(metrics):
runtime = metric.evaluate(y_test, predicted_scores,
accumulated_feature_time,
scenario.algorithm_cutoff_time)
approach_metric_values[i] = (approach_metric_values[i] + runtime)
# store runtimes on a per instance basis in ASLib format
runtime = simple_runtime_metric.evaluate(
y_test, predicted_scores, accumulated_feature_time,
scenario.algorithm_cutoff_time)
run_status_to_print = "ok" if runtime < scenario.algorithm_cutoff_time else "timeout"
line_to_store = test_scenario.instances[
instance_id] + ",1," + approach.get_name() + "," + str(
runtime) + "," + run_status_to_print
instancewise_result_strings.append(line_to_store)
write_instance_wise_results_to_file(instancewise_result_strings,
scenario.scenario)
approach_metric_values = np.true_divide(approach_metric_values,
num_counted_test_values)
for i, metric in enumerate(metrics):
print(metrics[i].get_name() +
': {0:.10f}'.format(approach_metric_values[i]))
return approach_metric_values
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Test models learned with train-as-auto-sklearn. It "
"writes the predictions to disk as a \"long\" data frame. The output "
"file is in gzipped csv format.")
parser.add_argument('scenario', help="The ASlib scenario")
parser.add_argument('model_template', help="A template string for the filenames for "
"the learned models. ${solver} and ${fold} are the template part of "
"the string. It is probably necessary to surround this argument with "
"single quotes in order to prevent shell replacement of the template "
"parts.")
parser.add_argument('out', help="The output csv file")
parser.add_argument('--config', help="A (yaml) config file which "
"specifies options controlling the learner behavior")
logging_utils.add_logging_options(parser)
args = parser.parse_args()
logging_utils.update_logging(args)
msg = "Loading ASlib scenario"
logger.info(msg)
scenario = ASlibScenario()
scenario.read_scenario(args.scenario)
if args.config is not None:
msg = "Loading yaml config file"
logger.info(msg)
config = yaml.load(open(args.config))
else:
config = {}
msg = "Creating string templates"
logger.info(msg)
model_template = string.Template(args.model_template)
msg = "Finding folds from ASlib scenario"
logger.info(msg)
folds = [int(i) for i in scenario.cv_data['fold'].unique()]
folds = sorted(folds)
msg = "Making predictions"
logger.info(msg)
all_predictions = []
it = itertools.product(scenario.algorithms, folds)
for solver, fold in it:
model_file = model_template.substitute(solver=solver, fold=fold)
if not os.path.exists(model_file):
msg = "Could not find model file. Skipping: {}".format(model_file)
logger.warning(msg)
continue
try:
model = joblib.load(model_file)
except:
msg = ("Problem loading the model file. Skipping: {}".format(
model_file))
logger.warning(msg)
continue
msg = "Processing. solver: {}. fold: {}".format(solver, fold)
logger.info(msg)
testing, training = scenario.get_split(fold)
y_pred = model.predict(testing.feature_data)
if 'log_performance_data':
# exp transform it back out
y_pred = np.expm1(y_pred)
pred_df = pd.DataFrame()
pred_df['instance_id'] = testing.feature_data.index
pred_df['solver'] = solver
pred_df['fold'] = fold
pred_df['actual'] = testing.performance_data[solver].values
pred_df['predicted'] = y_pred
all_predictions.append(pred_df)
msg = "Joining all predictions in a long data frame"
logger.info(msg)
all_predictions = pd.concat(all_predictions)
msg = "Writing predictions to disk"
logger.info(msg)
utils.write_df(all_predictions, args.out, index=False)
def evaluate_train_test_split(scenario: ASlibScenario, approach, metrics,
fold: int, amount_of_training_instances: int,
train_status: str):
test_scenario, train_scenario = scenario.get_split(indx=fold)
if train_status != 'all':
train_scenario = copy.deepcopy(train_scenario)
threshold = train_scenario.algorithm_cutoff_time
if train_status == 'clip_censored':
train_scenario.performance_data = train_scenario.performance_data.clip(
upper=threshold)
elif train_status == 'ignore_censored':
train_scenario.performance_data = train_scenario.performance_data.replace(
10 * threshold, np.nan)
if approach.get_name() == 'oracle':
approach.fit(test_scenario, fold, amount_of_training_instances)
else:
approach.fit(train_scenario, fold, amount_of_training_instances)
approach_metric_values = np.zeros(len(metrics))
num_counted_test_values = 0
feature_data = test_scenario.feature_data.to_numpy()
performance_data = test_scenario.performance_data.to_numpy()
feature_cost_data = test_scenario.feature_cost_data.to_numpy(
) if test_scenario.feature_cost_data is not None else None
for instance_id in range(0, len(test_scenario.instances)):
#logger.debug("Test instance " + str(instance_id) + "/" + str(len(test_scenario.instances)))
X_test = feature_data[instance_id]
y_test = performance_data[instance_id]
accumulated_feature_time = 0
if test_scenario.feature_cost_data is not None and approach.get_name(
) != 'sbs' and approach.get_name() != 'oracle':
feature_time = feature_cost_data[instance_id]
accumulated_feature_time = np.sum(feature_time)
contains_non_censored_value = False
for y_element in y_test:
if y_element < test_scenario.algorithm_cutoff_time:
contains_non_censored_value = True
if contains_non_censored_value:
num_counted_test_values += 1
predicted_scores = approach.predict(X_test, instance_id)
for i, metric in enumerate(metrics):
runtime = metric.evaluate(y_test, predicted_scores,
accumulated_feature_time,
scenario.algorithm_cutoff_time)
approach_metric_values[i] = (approach_metric_values[i] +
runtime)
approach_metric_values = np.true_divide(approach_metric_values,
num_counted_test_values)
print('PAR10: {0:.10f}'.format(approach_metric_values[0]))
return approach_metric_values
class ASAPy(object):
def __init__(self, output_dn: str = ".", plot_log_perf: bool = False):
'''
Constructor
Arguments
---------
output_dn:str
output directory name
'''
self.logger = logging.getLogger("ASAPy")
self.scenario = None
self.output_dn = output_dn
self.plot_log_perf = plot_log_perf
if not os.path.isdir(self.output_dn):
os.mkdir(self.output_dn)
def read_scenario_ASlib(self, scenario_dn: str):
'''
Read scenario from ASlib format
Arguments
---------
scenario_dn: str
Scenario directory name
'''
self.scenario = ASlibScenario()
self.scenario.read_scenario(dn=scenario_dn)
def read_scenario_CSV(self, csv_data: namedtuple):
'''
Read scenario from ASlib format
Arguments
---------
csv_data: namedtuple
namedtuple with the following fields: "perf_csv", "feat_csv", "obj", "cutoff", "maximize", "cv_csv"
"cv_csv" can be None
'''
self.scenario = ASlibScenario()
self.scenario.read_from_csv(perf_fn=csv_data.perf_csv,
feat_fn=csv_data.feat_csv,
objective=csv_data.obj,
runtime_cutoff=csv_data.cutoff,
maximize=csv_data.maximize,
cv_fn=csv_data.cv_csv)
def get_default_config(self):
'''
get default configuration which enables all plots
Returns
-------
dict
'''
config = {
"Performance Analysis": {
"Status bar plot": True,
"Box plot": True,
"Violin plot": True,
"CDF plot": True,
"Scatter plots": True,
"Correlation plot": True,
"Contribution of algorithms": True,
"Critical Distance Diagram": True,
"Footprints": True,
"Instance Hardness": True,
"Baselines": True
},
"Feature Analysis": {
"Status Bar Plot": True,
"Violin and box plots": True,
"Correlation plot": True,
"Feature importance": True,
"Clustering": True,
"CDF plot on feature costs": True
}
}
return config
def print_config(self):
'''
generate template for config file
'''
print(json.dumps(self.get_default_config(), indent=2))
def load_config(self, fn: str):
'''
load config from file
Arguments
---------
fn: str
file name with config in json format
Returns
-------
config: dict
'''
with open(fn) as fp:
config = json.load(fp)
return config
def main(self, config: dict, max_algos: int = 20, only_fold: int = None):
'''
main method
Arguments
---------
config: dict
configuration that enables or disables plots
max_algos: int
maximum number of algos to consider;
if more are available, we take the n best algorithm on average performance
only_fold: int
use only the given <only_fold> cv-fold data for analyze
'''
if only_fold:
self.logger.info("Using only test data from %d cv-split" %
(only_fold))
_, self.scenario = self.scenario.get_split(only_fold)
n_prev_algos = None
if self.scenario is None:
raise ValueError(
"Please first read in Scenario data; use scenario input or csv input"
)
if self.scenario.performance_type[
0] == "solution_quality" and self.scenario.maximize[0]:
# revoke inverting the performance as done in the scenario reader
self.scenario.performance_data *= -1
self.logger.info("Revoke * -1 on performance data")
if len(self.scenario.algorithms) > max_algos:
n_prev_algos = len(self.scenario.algorithms)
self.logger.warning(
"We reduce the algorithms to the greedy selected VBS-improving algorithms (at most %d)"
% (max_algos))
pa = PerformanceAnalysis(output_dn=self.output_dn,
scenario=self.scenario)
algos_score = pa.reduce_algos(max_algos=max_algos)
best_algos = [a[0] for a in algos_score]
self.scenario.algorithms = best_algos
self.scenario.performance_data = self.scenario.performance_data[
best_algos]
self.scenario.runstatus_data = self.scenario.runstatus_data[
best_algos]
data = OrderedDict()
# meta data
meta_data_df = self.get_meta_data()
data["Meta Data"] = {"table": meta_data_df.to_html(header=False)}
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# performance analysis
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
if config.get("Performance Analysis"):
pa = PerformanceAnalysis(output_dn=self.output_dn,
scenario=self.scenario)
data["Performance Analysis"] = OrderedDict()
if n_prev_algos is not None:
data["Performance Analysis"][
"tooltip"] = "To provide a clear overview, we reduced the number of algorithms (%d) to the greedily selected most VBS-improving %d algorithms." % (
n_prev_algos, len(self.scenario.algorithms))
if config["Performance Analysis"].get("Baselines"):
baseline_table = pa.get_baselines()
data["Performance Analysis"]["Baselines"] = {
"tooltip":
"Baslines: Best Single Algorithm (i.e., best algorithm on average across instances), Virtual Best Solver (aka Oracle, i.e., average across best performance per instance; theoretical best algorithm selector)",
"table": baseline_table
}
if config["Performance Analysis"].get("Status bar plot"):
status_plot = pa.get_bar_status_plot()
data["Performance Analysis"]["Status bar plot"] = {
"tooltip":
"Stacked bar plots for runstatus of each algorithm",
"figure": status_plot
}
# get box plot
if config["Performance Analysis"].get("Box plot"):
box_plot = pa.get_box_plots(plot_log_perf=self.plot_log_perf)
data["Performance Analysis"]["Box plot"] = {
"tooltip":
"Box plots to show the performance distribution of each algorithm",
"figure": box_plot
}
# get violin plot
if config["Performance Analysis"].get("Violin plot"):
violion_plot = pa.get_violin_plots(
plot_log_perf=self.plot_log_perf)
data["Performance Analysis"]["Violin plot"] = {
"tooltip":
"Violin plots to show the probablity density of each algorithm's performance. Also showing the median (middle line) and min/max value.",
"figure": violion_plot
}
# get cdf plot
if config["Performance Analysis"].get("CDF plot"):
cdf_plot = pa.get_cdf_plots(plot_log_perf=self.plot_log_perf)
data["Performance Analysis"]["CDF plot"] = {
"tooltip":
"Cumulative Distribution function (CDF) plots. At each point x (e.g., running time cutoff), how many of the instances (in percentage) can be solved. Better algorithms have a higher curve for minimization problems.",
"figure": cdf_plot
}
# get cd diagram
if config["Performance Analysis"].get("Critical Distance Diagram"):
cd_plot = pa.get_cd_diagram()
data["Performance Analysis"]["Critical Distance Diagram"] = {
"tooltip":
"Critical Distance (CD) diagram based on a Nemenyi two tailed test using average rankings. CD (top left) shows the critical distance. Distances larger than CD corresponds to a statistical significant difference in the ranking. We show only the best 20 ranked algorithms.",
"figure": cd_plot
}
# generate scatter plots
if config["Performance Analysis"].get("Scatter plots"):
scatter_plots = pa.scatter_plots(
plot_log_perf=self.plot_log_perf)
data["Performance Analysis"]["Scatter plots"] = OrderedDict({
"tooltip":
"Scatter plot to compare the performance of two algorithms on all instances -- each dot represents one instance."
})
scatter_plots = sorted(scatter_plots,
key=lambda x: x[0] + x[1])
for plot_tuple in scatter_plots:
key = "%s vs %s" % (plot_tuple[0], plot_tuple[1])
data["Performance Analysis"]["Scatter plots"][key] = {
"figure": plot_tuple[2]
}
# generate correlation plot
if config["Performance Analysis"].get("Correlation plot"):
correlation_plot = pa.correlation_plot()
data["Performance Analysis"]["Correlation plot"] = {
"tooltip":
"Correlation based on Spearman Correlation Coefficient between all algorithms and clustered with Wards hierarchical clustering approach. Darker fields corresponds to a larger correlation between the algorithms. See [Xu et al SAT 2012]",
"figure": correlation_plot
}
# get contribution values
if config["Performance Analysis"].get(
"Contribution of algorithms"):
avg_fn, marg_fn, shap_fn = pa.get_contribution_values()
data["Performance Analysis"][
"Contribution of algorithms"] = OrderedDict(
{"tooltip": "Contribution of each algorithm"})
data["Performance Analysis"]["Contribution of algorithms"][
"Average Performance"] = {
"figure": avg_fn
}
data["Performance Analysis"]["Contribution of algorithms"][
"Marginal Contribution"] = {
"figure":
marg_fn,
"tooltip":
"Marginal contribution to the virtual best solver (VBS, aka oracle) (i.e., how much decreases the VBS performance by removing the algorithm; higher value correspond to more importance). See [Xu et al SAT 2012]"
}
data["Performance Analysis"]["Contribution of algorithms"][
"Shapley Values"] = {
"figure":
shap_fn,
"tooltip":
"Shapley values (i.e., marginal contribution across all possible subsets of portfolios; again higher values corresponds to more importance; see [Frechette et al AAAI'16]. For running time scenarios, the metric is cutoff - running time; for non-running time, the metric is (worst performance across all instances and algorithms)- performance."
}
portfolio_table = pa.get_greedy_portfolio_constr()
data["Performance Analysis"]["Contribution of algorithms"][
"Portfolio Construction"] = {
"tooltip":
"Starting with the Single Best Solver, iteratively add algorithms to the portfolio by optimizing VBS",
"table": portfolio_table
}
# generate footprint plots
if config["Performance Analysis"].get("Footprints"):
footprints_plots = pa.get_footprints()
data["Performance Analysis"]["Footprints"] = OrderedDict({
"tooltip":
"Footprints of algorithms (instances red marked if the algorithm is at most 5% away from oracle performance) in 2-d PCA feature space. Inspired by [Smith-Miles et al. Computers & OR 2014]"
})
footprints_plots = sorted(footprints_plots, key=lambda x: x[0])
for plot_tuple in footprints_plots:
key = "%s" % (plot_tuple[0])
data["Performance Analysis"]["Footprints"][key] = {
"html": plot_tuple[1],
"figure": plot_tuple[2]
}
# generate instance hardness plot
if config["Performance Analysis"].get("Instance Hardness"):
hardness_plot = pa.instance_hardness()
data["Performance Analysis"]["Instance Hardness"] = {
"tooltip":
"Projecting instances into 2d PCA feature space; the color encodes the number of algorithms that perform within 5% of the oracle performance.",
"figure": hardness_plot[1],
"html": hardness_plot[0]
}
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# feature analysis
#>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
if config.get("Feature Analysis"):
data["Feature Analysis"] = OrderedDict()
fa = FeatureAnalysis(output_dn=self.output_dn,
scenario=self.scenario)
if config["Feature Analysis"].get("Status Bar Plot"):
status_plot = fa.get_bar_status_plot()
data["Feature Analysis"]["Status Bar Plot"] = {
"tooltip":
"Stacked bar plots for runstatus of each feature groupe",
"figure": status_plot
}
# box and violin plots
if config["Feature Analysis"].get("Violin and box plots"):
name_plots = fa.get_box_violin_plots()
data["Feature Analysis"]["Violin and box plots"] = OrderedDict({
"tooltip":
"Violin and Box plots to show the distribution of each instance feature. We removed NaN from the data."
})
for plot_tuple in name_plots:
key = "%s" % (plot_tuple[0])
data["Feature Analysis"]["Violin and box plots"][key] = {
"figure": plot_tuple[1]
}
# correlation plot
if config["Feature Analysis"].get("Correlation plot"):
correlation_plot = fa.correlation_plot()
data["Feature Analysis"]["Correlation plot"] = {
"tooltip":
"Correlation based on Pearson product-moment correlation coefficients between all features and clustered with Wards hierarchical clustering approach. Darker fields corresponds to a larger correlation between the features.",
"figure": correlation_plot
}
# feature importance
if config["Feature Analysis"].get("Feature importance"):
importance_plot = fa.feature_importance()
data["Feature Analysis"]["Feature importance"] = {
"tooltip":
"Using the approach of SATZilla'11, we train a cost-sensitive random forest for each pair of algorithms and average the feature importance (using gini as splitting criterion) across all forests. We show the median, 25th and 75th percentiles across all random forests of the 15 most important features.",
"figure": importance_plot
}
# cluster instances in feature space
if config["Feature Analysis"].get("Clustering"):
cluster_plot = fa.cluster_instances()
data["Feature Analysis"]["Clustering"] = {
"tooltip":
"Clustering instances in 2d; the color encodes the cluster assigned to each cluster. Similar to ISAC, we use a k-means to cluster the instances in the feature space. As pre-processing, we use standard scaling and a PCA to 2 dimensions. To guess the number of clusters, we use the silhouette score on the range of 2 to 12 in the number of clusters",
"figure": cluster_plot
}
# get cdf plot
if self.scenario.feature_cost_data is not None and config[
"Feature Analysis"].get("CDF plot on feature costs"):
cdf_plot = fa.get_feature_cost_cdf_plot()
data["Feature Analysis"]["CDF plot on feature costs"] = {
"tooltip":
"Cumulative Distribution function (CDF) plots. At each point x (e.g., running time cutoff), for how many of the instances (in percentage) have we computed the instance features. Faster feature computation steps have a higher curve. Missing values are imputed with the maximal value (or running time cutoff).",
"figure": cdf_plot
}
self.create_html(data=data)
def create_html(self, data: OrderedDict):
'''
create html report
'''
html_builder = HTMLBuilder(output_dn=self.output_dn,
scenario_name=self.scenario.scenario)
html_builder.generate_html(data)
def get_meta_data(self):
'''
read meta data from self.scenario and generate a pandas.Dataframe with it
'''
data = []
data.append(("Number of instances", len(self.scenario.instances)))
data.append(("Number of algorithms", len(self.scenario.algorithms)))
data.append(
("Performance measure", self.scenario.performance_measure[0]))
data.append(("Performance type", self.scenario.performance_type[0]))
data.append(("Maximize?", str(self.scenario.maximize[0])))
if self.scenario.algorithm_cutoff_time:
data.append(("Running time cutoff (algorithm)",
str(self.scenario.algorithm_cutoff_time)))
if self.scenario.algorithm_cutoff_memory:
data.append(("Memory cutoff (algorithm)",
str(self.scenario.algorithm_cutoff_memory)))
if self.scenario.features_cutoff_time:
data.append(("Running time cutoff (features)",
str(self.scenario.features_cutoff_time)))
if self.scenario.features_cutoff_memory:
data.append(("Memory cutoff (Features)",
str(self.scenario.features_cutoff_memory)))
data.append(("# Deterministic features",
len(self.scenario.features_deterministic)))
data.append(
("# Stochastic features", len(self.scenario.features_stochastic)))
data.append(("# Feature groups", len(self.scenario.feature_steps)))
data.append(("# Deterministic algorithms",
len(self.scenario.algortihms_deterministics)))
data.append(("# Stochastic algorithms",
len(self.scenario.algorithms_stochastic)))
if self.scenario.feature_cost_data is not None:
data.append(("Feature costs provided?", "True"))
else:
data.append(("Feature costs provided?", "False"))
meta_data = pd.DataFrame(data=list(map(lambda x: x[1], data)),
index=list(map(lambda x: x[0], data)),
columns=[""])
return meta_data
