def _init_pimp_and_validator(self, rh, alternative_output_dir=None):
"""Create ParameterImportance-object and use it's trained model for validation and further predictions
We pass validated runhistory, so that the returned model will be based on as much information as possible
Parameters
----------
rh: RunHistory
runhistory used to build EPM
alternative_output_dir: str
e.g. for budgets we want pimp to use an alternative output-dir (subfolders per budget)
"""
self.logger.debug(
"Using '%s' as output for pimp", alternative_output_dir
if alternative_output_dir else self.output_dir)
self.pimp = Importance(
scenario=copy.deepcopy(self.scenario),
runhistory=rh,
incumbent=self.default, # Inject correct incumbent later
parameters_to_evaluate=4,
save_folder=alternative_output_dir
if alternative_output_dir else self.output_dir,
seed=self.rng.randint(1, 100000),
max_sample_size=self.pimp_max_samples,
fANOVA_pairwise=self.fanova_pairwise,
preprocess=False)
self.model = self.pimp.model
# Validator (initialize without trajectory)
self.validator = Validator(self.scenario, None, None)
self.validator.epm = self.model
def execute(save_folder,
runhistory_location,
configspace_location,
modus='ablation',
seed=1):
with open(runhistory_location, 'r') as runhistory_filep:
runhistory = json.load(runhistory_filep)
# create scenario file
scenario_dict = {
'run_obj': 'quality',
'deterministic': 1,
'paramfile': configspace_location
}
trajectory_lines = openmlpimp.utils.runhistory_to_trajectory(
runhistory, maximize=True)
if len(trajectory_lines) != 1:
raise ValueError('trajectory file should containexactly one line.')
traj_file = tempfile.NamedTemporaryFile('w', delete=False)
for line in trajectory_lines:
json.dump(line, traj_file)
traj_file.write("\n")
traj_file.close()
num_params = len(trajectory_lines[0]['incumbent'])
importance = Importance(scenario_dict,
runhistory_file=runhistory_location,
parameters_to_evaluate=num_params,
traj_file=traj_file.name,
seed=seed,
save_folder=save_folder)
try:
os.makedirs(save_folder)
except FileExistsError:
pass
for i in range(5):
try:
result = importance.evaluate_scenario(modus)
filename = 'pimp_values_%s.json' % modus
with open(os.path.join(save_folder, filename),
'w') as out_file:
json.dump(result,
out_file,
sort_keys=True,
indent=4,
separators=(',', ': '))
importance.plot_results(name=os.path.join(save_folder, modus),
show=False)
return save_folder + "/" + filename
except ZeroDivisionError as e:
pass
raise e
def _init_pimp_and_validator(
self,
alternative_output_dir=None,
):
"""Create ParameterImportance-object and use it's trained model for validation and further predictions.
We pass a combined (original + validated) runhistory, so that the returned model will be based on as much
information as possible
Parameters
----------
alternative_output_dir: str
e.g. for budgets we want pimp to use an alternative output-dir (subfolders per budget)
"""
self.logger.debug(
"Using '%s' as output for pimp", alternative_output_dir
if alternative_output_dir else self.output_dir)
self.pimp = Importance(
scenario=copy.deepcopy(self.scenario),
runhistory=self.combined_runhistory,
incumbent=self.incumbent if self.incumbent else self.default,
save_folder=alternative_output_dir
if alternative_output_dir is not None else self.output_dir,
seed=self.rng.randint(1, 100000),
max_sample_size=self.options['fANOVA'].getint("pimp_max_samples"),
fANOVA_pairwise=self.options['fANOVA'].getboolean(
"fanova_pairwise"),
preprocess=False,
verbose=1, # disable progressbars
)
# Validator (initialize without trajectory)
self.validator = Validator(self.scenario, None, None)
self.validator.epm = self.pimp.model
def parameter_importance(self,
modus,
incumbent,
output,
num_params=4,
num_pairs=0):
"""Calculate parameter-importance using the PIMP-package.
Currently ablation, forward-selection and fanova are used.
Parameters
----------
modus: str
modus for parameter importance, from [forward-selection, ablation,
fanova]
Returns
-------
importance: pimp.Importance
importance object with evaluated data
"""
self.logger.info("... parameter importance {}".format(modus))
# Evaluate parameter importance
save_folder = output
if not self.pimp:
self.pimp = Importance(scenario=copy.deepcopy(self.scenario),
runhistory=self.original_rh,
incumbent=incumbent,
parameters_to_evaluate=num_params,
save_folder=save_folder,
seed=12345,
max_sample_size=self.max_pimp_samples,
fANOVA_pairwise=self.fanova_pairwise,
preprocess=False)
result = self.pimp.evaluate_scenario([modus], save_folder)
self.evaluators.append(self.pimp.evaluator)
return self.pimp
def __init__(self,
scenario: Scenario,
smac: Union[SMAC, None] = None,
mode: str = 'all',
X: Union[None, List[list], np.ndarray] = None,
y: Union[None, List[list], np.ndarray] = None,
numParams: int = -1,
impute: bool = False,
seed: int = 12345,
run: bool = False,
max_sample_size: int = -1,
fanova_cut_at_default: bool = False,
fANOVA_pairwise: bool = True,
forwardsel_feat_imp: bool = False,
incn_quant_var: bool = True,
marginalize_away_instances: bool = False,
save_folder: str = 'PIMP'):
"""
Interface to be used with SMAC or with X and y matrices.
:param scenario: The scenario object, that knows the configuration space.
:param smac: The smac object that keeps all the run-data
:param mode: The mode with which to run PIMP [ablation, fanova, all, forward-selection]
:param X: Numpy Array that contains parameter arrays
:param y: Numpy array that contains the corresponding performance values
:param numParams: The number of parameters to evaluate
:param impute: Flag to decide if censored data gets imputed or not
:param seed: The random seed
:param run: Flag to immediately compute the importance values after this setup or not.
"""
self.scenario = scenario
self.imp = None
self.mode = mode
self.save_folder = save_folder
if not os.path.exists(self.save_folder): os.mkdir(self.save_folder)
if smac is not None:
self.imp = Importance(scenario=scenario,
runhistory=smac.runhistory,
incumbent=smac.solver.incumbent,
seed=seed,
parameters_to_evaluate=numParams,
save_folder='PIMP',
impute_censored=impute,
max_sample_size=max_sample_size,
fANOVA_cut_at_default=fanova_cut_at_default,
fANOVA_pairwise=fANOVA_pairwise,
forwardsel_feat_imp=forwardsel_feat_imp,
incn_quant_var=incn_quant_var,
preprocess=marginalize_away_instances)
elif X is not None and y is not None:
X = np.array(X)
y = np.array(y)
runHist = RunHistory(average_cost)
if X.shape[0] != y.shape[0]:
raise Exception('Number of samples in X and y dont match!')
n_params = len(scenario.cs.get_hyperparameters())
feats = None
if X.shape[1] > n_params:
feats = X[:, n_params:]
assert feats.shape[1] == scenario.feature_array.shape[1]
X = X[:, :n_params]
for p in range(X.shape[1]): # Normalize the data to fit into [0, 1]
_min, _max = np.min(X[:, p]), np.max(X[:, p])
if _min < 0. or 1 < _max: # if it is not already normalized
for id, v in enumerate(X[:, p]):
X[id, p] = (v - _min) / (_max - _min)
# Add everything to a runhistory such that PIMP can work with it
for x, feat, y_val in zip(X, feats if feats is not None else X, y):
id = None
for inst in scenario.feature_dict: # determine on which instance a configuration was run
if np.all(scenario.feature_dict[inst] == feat):
id = inst
break
runHist.add(Configuration(scenario.cs, vector=x), y_val, 0, StatusType.SUCCESS, id)
self.X = X
self.y = y
best_ = None # Determine incumbent according to the best mean cost in the runhistory
for config in runHist.config_ids:
inst_seed_pairs = runHist.get_runs_for_config(config)
all_ = []
for inst, seed in inst_seed_pairs:
rk = RunKey(runHist.config_ids[config], inst, seed)
all_.append(runHist.data[rk].cost)
mean = np.mean(all_)
if best_ is None or best_[0] > mean:
best_ = (mean, config)
incumbent = best_[1]
self.imp = Importance(scenario=scenario,
runhistory=runHist,
seed=seed,
parameters_to_evaluate=numParams,
save_folder=self.save_folder,
impute_censored=impute,
incumbent=incumbent,
fANOVA_cut_at_default=fanova_cut_at_default,
fANOVA_pairwise=fANOVA_pairwise,
forwardsel_feat_imp=forwardsel_feat_imp,
incn_quant_var=incn_quant_var,
preprocess=marginalize_away_instances
)
else:
raise Exception('Neither X and y matrices nor a SMAC object were specified to compute the importance '
'values from!')
if run:
self.compute_importances()
def cmd_line_call():
"""
Main Parameter importance script.
"""
cmd_reader = CMDs()
args, misc_ = cmd_reader.read_cmd() # read cmd args
cwd = os.path.abspath(os.getcwd())
if args.out_folder and not os.path.isabs(args.out_folder):
args.out_folder = os.path.abspath(args.out_folder)
if args.trajectory and not os.path.isabs(args.trajectory):
args.trajectory = os.path.abspath(args.trajectory)
if not os.path.isabs(args.scenario_file):
args.scenario_file = os.path.abspath(args.scenario_file)
if not os.path.isabs(args.history):
args.history = os.path.abspath(args.history)
os.chdir(args.wdir)
logging.basicConfig(level=args.verbose_level)
ts = time.time()
ts = datetime.datetime.fromtimestamp(ts).strftime('%Y_%m_%d_%H:%M:%S')
fanova_ready = True
try:
import fanova
except ImportError:
warnings.simplefilter('always', ImportWarning)
warnings.warn('fANOVA is not installed in your environment. To install it please run '
'"git+http://github.com/automl/fanova.git@master"')
fanova_ready = False
if 'influence-model' in args.modus:
logging.warning('influence-model not fully supported yet!')
if 'incneighbor' in args.modus:
warnings.simplefilter('always', DeprecationWarning)
warnings.warn('incneighbor will be deprecated in version 1.0.0 as it was the development name of'
' lpi. Use lpi instead.', DeprecationWarning, stacklevel=2)
if 'lpi' in args.modus: # LPI will replace incneighbor in the future
args.modus[args.modus.index('lpi')] = 'incneighbor'
if 'fanova' in args.modus and not fanova_ready:
raise ImportError('fANOVA is not installed! To install it please run '
'"git+http://github.com/automl/fanova.git@master"')
if 'all' in args.modus:
choices = ['ablation',
'forward-selection',
'fanova',
'incneighbor']
if not fanova_ready:
raise ImportError('fANOVA is not installed! To install it please run '
'"git+http://github.com/automl/fanova.git@master"')
del args.modus[args.modus.index('all')]
if len(args.modus) == len(choices):
pass
else:
args.modus = choices
if not args.out_folder:
if len(args.modus) > 1:
tmp = ['all']
else:
tmp = args.modus
if 'incneighbor' in args.modus:
tmp = ['lpi']
save_folder = os.path.join(cwd, 'PIMP_%s' % '_'.join(tmp))
if os.path.exists(os.path.abspath(save_folder)):
save_folder = os.path.join(cwd, 'PIMP_%s_%s' % ('_'.join(tmp), ts))
else:
if len(args.modus) > 1:
tmp = ['all']
else:
tmp = args.modus
if 'incneighbor' in args.modus:
tmp = ['lpi']
if os.path.exists(os.path.abspath(args.out_folder)) or os.path.exists(os.path.abspath(
args.out_folder + '_%s' % '_'.join(tmp))):
save_folder = os.path.join(cwd, args.out_folder + '_%s_%s' % ('_'.join(tmp), ts))
else:
save_folder = os.path.join(cwd, args.out_folder + '_%s' % '_'.join(tmp))
importance = Importance(scenario_file=args.scenario_file,
runhistory_file=args.history,
parameters_to_evaluate=args.num_params,
traj_file=args.trajectory, seed=args.seed,
save_folder=save_folder,
impute_censored=args.impute,
max_sample_size=args.max_sample_size,
fANOVA_cut_at_default=args.fanova_cut_at_default,
fANOVA_pairwise=args.fanova_pairwise,
forwardsel_feat_imp=args.forwardsel_feat_imp,
incn_quant_var=args.incn_quant_var,
preprocess=args.marg_inst,
forwardsel_cv=args.forwardsel_cv) # create importance object
with open(os.path.join(save_folder, 'pimp_args.json'), 'w') as out_file:
json.dump(args.__dict__, out_file, sort_keys=True, indent=4, separators=(',', ': '))
result = importance.evaluate_scenario(args.modus, save_folder=save_folder)
if args.table:
importance.table_for_comparison(evaluators=result[1], name=os.path.join(
save_folder, 'pimp_table_%s.tex' % args.modus), style='latex')
else:
importance.table_for_comparison(evaluators=result[1], style='cmd')
os.chdir(cwd)
if __name__ == '__main__':
"""
Main Parameter importance script.
"""
cmd_reader = CMDs()
args, misc_ = cmd_reader.read_cmd() # read cmd args
logging.basicConfig(level=args.verbose_level)
ts = time.time()
ts = datetime.datetime.fromtimestamp(ts).strftime('%Y_%m_%d_%H:%M:%S')
save_folder = 'PIMP_%s_%s' % (args.modus, ts)
importance = Importance(
scenario_file=args.scenario_file,
runhistory_file=args.history,
parameters_to_evaluate=args.num_params,
traj_file=args.trajectory,
seed=args.seed,
save_folder=save_folder,
impute_censored=args.impute) # create importance object
save_folder += '_run1'
os.makedirs(save_folder, exist_ok=True)
with open(os.path.join(save_folder, 'pimp_args.json'), 'w') as out_file:
json.dump(args.__dict__,
out_file,
sort_keys=True,
indent=4,
separators=(',', ': '))
result = importance.evaluate_scenario(args.modus, save_folder=save_folder)
if args.modus == 'all':
with open(
class Analyzer(object):
"""
This class serves as an interface to all the individual analyzing and
plotting components. The plotter object is responsible for the actual
plotting of things, but should not be invoked via the facade (which is
constructed for cmdline-usage).
"""
def __init__(self,
original_rh,
validated_rh,
default,
incumbent,
train_test,
scenario,
validator,
output,
max_pimp_samples,
fanova_pairwise=True):
"""
Parameters
----------
original_rh: RunHistory
runhistory containing all runs that have actually been run
validated_rh: RunHistory
runhistory containing all runs from original_rh + estimates for
default and all incumbents for all instances
default, incumbent: Configuration
default and overall incumbent
train_test: bool
whether is distinction is made (in cdf and scatter)
scenario: Scenario
the scenario object
validator: Validator
validator object (to estimate using EPM)
output: string
output-directory
"""
self.logger = logging.getLogger("cave.analyzer")
# Important objects for analysis
self.original_rh = original_rh
self.validated_rh = validated_rh
self.default = default
self.incumbent = incumbent
self.train_test = train_test
self.scenario = scenario
self.validator = validator
self.pimp = None # PIMP object for reuse
self.feat_analysis = None # feat_analysis object for reuse
self.evaluators = []
self.output = output
self.importance = None # Used to store dictionary containing parameter
# importances, so it can be used by analysis
self.feat_importance = None # Used to store dictionary w feat_imp
conf1_runs = get_cost_dict_for_config(self.validated_rh, self.default)
conf2_runs = get_cost_dict_for_config(self.validated_rh,
self.incumbent)
self.plotter = Plotter(self.scenario,
self.train_test,
conf1_runs,
conf2_runs,
output=self.output)
self.max_pimp_samples = max_pimp_samples
self.fanova_pairwise = fanova_pairwise
def get_timeouts(self, config):
""" Get number of timeouts in config per runs in total (not per
instance)
Parameters
----------
config: Configuration
configuration from which to calculate the timeouts
Returns
-------
timeouts: tuple(int, int)
tuple (timeouts, total runs)
"""
cutoff = self.scenario.cutoff
timeouts = get_timeout(self.validated_rh, config, cutoff)
if self.train_test:
if not cutoff:
return (("N", "A"), ("N", "A"))
train_timeout = len([
i for i in timeouts
if (timeouts[i] == False and i in self.scenario.train_insts)
])
test_timeout = len([
i for i in timeouts
if (timeouts[i] == False and i in self.scenario.test_insts)
])
return ((train_timeout, len(self.scenario.train_insts)),
(test_timeout, len(self.scenario.test_insts)))
else:
if not cutoff:
return ("N", "A")
timeout = len([i for i in timeouts if timeouts[i] == False])
no_timeout = len([i for i in timeouts if timeouts[i] == True])
return (timeout, no_timeout)
def get_parX(self, config, par=10):
"""Calculate parX-values of default and incumbent configs.
First determine PAR-timeouts for each run on each instances,
Second average over train/test if available, else just average.
Parameters
----------
config: Configuration
config to be calculated
par: int
par-factor to use
Returns
-------
(train, test) OR average -- tuple<float, float> OR float
PAR10 values for train- and test-instances, if available as tuple
else the general average
"""
runs = get_cost_dict_for_config(self.validated_rh, config)
# Penalize
if self.scenario.cutoff:
runs = [(k, runs[k]) if runs[k] < self.scenario.cutoff else
(k, self.scenario.cutoff * par) for k in runs]
else:
runs = [(k, runs[k]) for k in runs]
self.logger.info("Calculating penalized average runtime without "
"cutoff...")
# Average
if self.train_test:
train = np.mean(
[c for i, c in runs if i in self.scenario.train_insts])
test = np.mean(
[c for i, c in runs if i in self.scenario.test_insts])
return (train, test)
else:
return np.mean([c for i, c in runs])
####################################### TABLES #######################################
def create_overview_table(self, best_folder):
""" Create overview-table.
Parameters
----------
best_folder: str
path to folder/run with best incumbent
Returns
-------
table: str
overview table in HTML
"""
overview = OrderedDict([
('Run with best incumbent', best_folder),
('# Train instances', len(self.scenario.train_insts)),
('# Test instances', len(self.scenario.test_insts)),
('# Parameters', len(self.scenario.cs.get_hyperparameters())),
('Cutoff', self.scenario.cutoff),
('Walltime budget', self.scenario.wallclock_limit),
('Runcount budget', self.scenario.ta_run_limit),
('CPU budget', self.scenario.algo_runs_timelimit),
('Deterministic', self.scenario.deterministic),
])
# Split into two columns
overview_split = self._split_table(overview)
# Convert to HTML
df = DataFrame(data=overview_split)
table = df.to_html(escape=False,
header=False,
index=False,
justify='left')
return table
def create_performance_table(self, default, incumbent):
"""Create table, compare default against incumbent on train-,
test- and combined instances. Listing PAR10, PAR1 and timeouts.
Distinguishes between train and test, if available."""
self.logger.info("... create performance table")
def_timeout, inc_timeout = self.get_timeouts(
default), self.get_timeouts(incumbent)
def_par10, inc_par10 = self.get_parX(default,
10), self.get_parX(incumbent, 10)
def_par1, inc_par1 = self.get_parX(default,
1), self.get_parX(incumbent, 1)
dec_place = 3
if self.train_test:
# Distinction between train and test
# Create table
array = np.array([[
round(def_par10[0], dec_place),
round(def_par10[1], dec_place),
round(inc_par10[0], dec_place),
round(inc_par10[1], dec_place)
],
[
round(def_par1[0], dec_place),
round(def_par1[1], dec_place),
round(inc_par1[0], dec_place),
round(inc_par1[1], dec_place)
],
[
"{}/{}".format(def_timeout[0][0],
def_timeout[0][1]),
"{}/{}".format(def_timeout[1][0],
def_timeout[1][1]),
"{}/{}".format(inc_timeout[0][0],
inc_timeout[0][1]),
"{}/{}".format(inc_timeout[1][0],
inc_timeout[1][1])
]])
df = DataFrame(data=array,
index=['PAR10', 'PAR1', 'Timeouts'],
columns=['Train', 'Test', 'Train', 'Test'])
table = df.to_html()
# Insert two-column-header
table = table.split(sep='</thead>', maxsplit=1)[1]
new_table = "<table border=\"3\" class=\"dataframe\">\n"\
" <col>\n"\
" <colgroup span=\"2\"></colgroup>\n"\
" <colgroup span=\"2\"></colgroup>\n"\
" <thead>\n"\
" <tr>\n"\
" <td rowspan=\"2\"></td>\n"\
" <th colspan=\"2\" scope=\"colgroup\">Default</th>\n"\
" <th colspan=\"2\" scope=\"colgroup\">Incumbent</th>\n"\
" </tr>\n"\
" <tr>\n"\
" <th scope=\"col\">Train</th>\n"\
" <th scope=\"col\">Test</th>\n"\
" <th scope=\"col\">Train</th>\n"\
" <th scope=\"col\">Test</th>\n"\
" </tr>\n"\
"</thead>\n"
table = new_table + table
else:
# No distinction between train and test
array = np.array(
[[round(def_par10, dec_place),
round(inc_par10, dec_place)],
[round(def_par1, dec_place),
round(inc_par1, dec_place)],
[
"{}/{}".format(def_timeout[0], def_timeout[1]),
"{}/{}".format(inc_timeout[0], inc_timeout[1])
]])
df = DataFrame(data=array,
index=['PAR10', 'PAR1', 'Timeouts'],
columns=['Default', 'Incumbent'])
table = df.to_html()
self.performance_table = table
return table
def config_to_html(self, default: Configuration, incumbent: Configuration):
"""Create HTML-table to compare Configurations.
Removes unused parameters.
Parameters
----------
default, incumbent: Configurations
configurations to be converted
Returns
-------
table: str
HTML-table comparing default and incumbent
"""
# Remove unused parameters
keys = [k for k in default.keys() if default[k] or incumbent[k]]
default = [
default[k] if default[k] != None else "inactive" for k in keys
]
incumbent = [
incumbent[k] if incumbent[k] != None else "inactive" for k in keys
]
table = list(zip(keys, default, incumbent))
# Show first parameters that changed
same = [x for x in table if x[1] == x[2]]
diff = [x for x in table if x[1] != x[2]]
table = []
if len(diff) > 0:
table.extend([("-------------- Changed parameters: "\
"--------------", "-----", "-----")])
table.extend(diff)
if len(same) > 0:
table.extend([("-------------- Unchanged parameters: "\
"--------------", "-----", "-----")])
table.extend(same)
keys, table = [k[0] for k in table], [k[1:] for k in table]
df = DataFrame(data=table,
columns=["Default", "Incumbent"],
index=keys)
table = df.to_html()
return table
def _split_table(self, table: OrderedDict):
"""Splits an OrderedDict into a list of tuples that can be turned into a
HTML-table with pandas DataFrame
Parameters
----------
table: OrderedDict
table that is to be split into two columns
Returns
-------
table_split: List[tuple(key, value, key, value)]
list with two key-value pairs per entry that can be used by pandas
df.to_html()
"""
table_split = []
keys = list(table.keys())
half_size = len(keys) // 2
for i in range(half_size):
j = i + half_size
table_split.append(("<b>" + keys[i] + "</b>", table[keys[i]],
"<b>" + keys[j] + "</b>", table[keys[j]]))
if len(keys) % 2 == 1:
table_split.append(
("<b>" + keys[-1] + "</b>", table[keys[-1]], '', ''))
return table_split
####################################### PARAMETER IMPORTANCE #######################################
def fanova(self,
incumbent,
num_params=10,
num_pairs=0,
marginal_threshold=0.05):
"""Wrapper for parameter_importance to save the importance-object/
extract the results. We want to show the top X most important
parameter-fanova-plots.
Parameters
----------
incumbent: Configuration
incumbent configuration
num_params: int
how many of the top important parameters should be shown
num_pairs: int (NOT WORKING)
for how many parameters pairwise marginals are plotted
n parameters -> n^2 plots
marginal_threshold: float
parameter/s must be at least this important to be mentioned
Returns
-------
fanova_table: str
html table with importances for all parameters
plots: Dict[str: st]
dictionary mapping single parameters to their plots
"""
self.parameter_importance("fanova",
incumbent,
self.output,
num_params,
num_pairs=num_pairs)
parameter_imp = self.pimp.evaluator.evaluated_parameter_importance
# Split single and pairwise (pairwise are string: "['p1','p2']")
pairwise_imp = {
k: v
for k, v in parameter_imp.items() if k.startswith("[")
}
for k in pairwise_imp.keys():
parameter_imp.pop(k)
# Set internal parameter importance for further analysis (such as
# parallel coordinates)
self.logger.debug("Fanova importance: %s", str(parameter_imp))
self.importance = parameter_imp
# Dicts to lists of tuples, sorted descending after importance and only
# including marginals > 0.05
parameter_imp = [(k, v) for k, v in sorted(
parameter_imp.items(), key=operator.itemgetter(1), reverse=True)
if v > 0.05]
pairwise_imp = [(k, v) for k, v in sorted(
pairwise_imp.items(), key=operator.itemgetter(1), reverse=True)
if v > 0.05]
# Create table
table = []
if len(parameter_imp) > 0:
table.extend([(20 * "-" + " Single importance: " + 20 * "-",
20 * "-")])
table.extend(parameter_imp)
if len(pairwise_imp) > 0:
table.extend([(20 * "-" + " Pairwise importance: " + 20 * "-",
20 * "-")])
# TODO assuming (current) form of "['param1','param2']", but not
# expecting it stays this way (on PIMPs side)
table.extend([(' & '.join(
[tmp.strip('\' ') for tmp in k.strip('[]').split(',')]), v)
for k, v in pairwise_imp])
keys, fanova_table = [k[0] for k in table], [k[1:] for k in table]
df = DataFrame(data=fanova_table, index=keys)
fanova_table = df.to_html(escape=False,
header=False,
index=True,
justify='left')
single_plots = {}
for p, v in parameter_imp:
single_plots[p] = os.path.join(self.output, "fanova", p + '.png')
# Check for pairwise plots
# Right now no way to access paths of the plots -> file issue
pairwise_plots = {}
for p, v in pairwise_imp:
p_new = p.replace('\'', '')
potential_path = os.path.join(self.output, 'fanova',
p_new + '.png')
self.logger.debug("Check for %s", potential_path)
if os.path.exists(potential_path):
pairwise_plots[p] = potential_path
return fanova_table, single_plots, pairwise_plots
def local_epm_plots(self):
plots = OrderedDict([])
if self.importance:
self.parameter_importance("incneighbor",
self.incumbent,
self.output,
num_params=3)
for p, i in [(k, v) for k, v in sorted(self.importance.items(),
key=operator.itemgetter(1),
reverse=True) if v > 0.05]:
plots[p] = os.path.join(self.output, 'incneighbor', p + '.png')
else:
self.logger.warning("Need to run fANOVA before incneighbor!")
raise ValueError()
return plots
def parameter_importance(self,
modus,
incumbent,
output,
num_params=4,
num_pairs=0):
"""Calculate parameter-importance using the PIMP-package.
Currently ablation, forward-selection and fanova are used.
Parameters
----------
modus: str
modus for parameter importance, from [forward-selection, ablation,
fanova]
Returns
-------
importance: pimp.Importance
importance object with evaluated data
"""
self.logger.info("... parameter importance {}".format(modus))
# Evaluate parameter importance
save_folder = output
if not self.pimp:
self.pimp = Importance(scenario=copy.deepcopy(self.scenario),
runhistory=self.original_rh,
incumbent=incumbent,
parameters_to_evaluate=num_params,
save_folder=save_folder,
seed=12345,
max_sample_size=self.max_pimp_samples,
fANOVA_pairwise=self.fanova_pairwise,
preprocess=False)
result = self.pimp.evaluate_scenario([modus], save_folder)
self.evaluators.append(self.pimp.evaluator)
return self.pimp
####################################### FEATURE IMPORTANCE #######################################
def feature_importance(self):
self.logger.info("... plotting feature importance")
forward_selector = FeatureForwardSelector(self.scenario,
self.original_rh)
imp = forward_selector.run()
self.logger.debug("FEAT IMP %s", imp)
self.feat_importance = imp
plots = forward_selector.plot_result(
os.path.join(self.output, 'feature_plots/importance'))
return (imp, plots)
####################################### PLOTS #######################################
def plot_parallel_coordinates(self, n_param=10, n_configs=500):
""" Creates a parallel coordinates plot visualizing the explored
parameter configuration space. """
self.logger.info("... plotting parallel coordinates")
# If a parameter importance has been performed in this analyzer-object,
# only plot the n_param most important parameters.
if self.importance:
n_param = min(
n_param,
max(3, len([x for x in self.importance.values() if x > 0.05])))
params = list(self.importance.keys())[:n_param]
else:
# TODO what if no parameter importance has been performed?
# plot all? random subset? -> atm: random
self.logger.info(
"No parameter importance performed. Plotting random "
"parameters in parallel coordinates plot.")
params = list(self.default.keys())[:n_param]
self.logger.info(
" plotting %s parameters for (max) %s configurations",
len(params), n_configs)
rh = self.original_rh if self.plotter.vizrh is None else self.plotter.vizrh
path = self.plotter.plot_parallel_coordinates(rh, self.output, params,
n_configs,
self.validator)
return path
def plot_cdf(self):
self.logger.info("... plotting eCDF")
cdf_path = os.path.join(self.output, 'cdf')
return self.plotter.plot_cdf_compare(output_fn_base=cdf_path)
def plot_scatter(self):
self.logger.info("... plotting scatter")
scatter_path = os.path.join(self.output, 'scatter')
return self.plotter.plot_scatter(output_fn_base=scatter_path)
@timing
def plot_confviz(self, incumbents, runhistories, max_confs=1000):
""" Plot the visualization of configurations, highlightning the
incumbents. Using original rh, so the explored configspace can be
estimated.
Parameters
----------
incumbents: List[Configuration]
list with incumbents, so they can be marked in plot
runhistories: List[RunHistory]
list of runhistories, so they can be marked in plot
max_confs: int
maximum number of data-points to plot
Returns
-------
confviz: str
script to generate the interactive html
"""
self.logger.info("... visualizing explored configspace")
confviz = self.plotter.visualize_configs(self.scenario,
runhistories=runhistories,
incumbents=incumbents,
max_confs_plot=max_confs)
return confviz
@timing
def plot_cost_over_time(self, traj, validator):
path = os.path.join(self.output, 'cost_over_time.png')
self.logger.info("... cost over time:")
self.logger.info(" plotting!")
self.plotter.plot_cost_over_time(self.validated_rh,
traj,
output=path,
validator=validator)
return path
@timing
def plot_algorithm_footprint(self,
algorithms=None,
density=200,
purity=0.95):
if not algorithms:
algorithms = {self.default: "default", self.incumbent: "incumbent"}
self.logger.info("... algorithm footprints:")
self.logger.info(" for: {}".format(algorithms.values()))
footprint = AlgorithmFootprint(self.validated_rh,
self.scenario.feature_dict, algorithms,
self.scenario.cutoff, self.output)
# Calculate footprints
#for i in range(100):
# for a in algorithms:
# footprint.footprint(a, 20, 0.95)
# Plot footprints
plots = footprint.plot_points_per_cluster()
return plots
####################################### FEATURE ANALYSIS #######################################
def feature_analysis(
self,
mode,
feat_names,
):
"""Use asapys feature analysis.
Parameters
----------
mode: str
from [box_violin, correlation, clustering]
Returns
-------
Corresponding plot paths
"""
self.logger.info("... feature analysis: %s", mode)
self.feat_analysis = FeatureAnalysis(
output_dn=self.output,
scenario=self.scenario,
feat_names=feat_names,
feat_importance=self.feat_importance)
if mode == 'box_violin':
return self.feat_analysis.get_box_violin_plots()
if mode == 'correlation':
self.feat_analysis.correlation_plot()
return self.feat_analysis.correlation_plot(imp=False)
if mode == 'clustering':
return self.feat_analysis.cluster_instances()
class CAVE(object):
def __init__(self,
folders: typing.List[str],
output_dir: str,
ta_exec_dir: typing.List[str],
file_format: str = 'SMAC3',
validation_format='NONE',
validation_method: str = 'epm',
pimp_max_samples: int = -1,
fanova_pairwise: bool = True,
use_budgets: bool = False,
seed: int = 42):
"""
Initialize CAVE facade to handle analyzing, plotting and building the report-page easily.
During initialization, the analysis-infrastructure is built and the data is validated, the overall best
incumbent is found and default+incumbent are evaluated for all instances for all runs, by default using an EPM.
In the internal data-management the we have three types of runhistories: *original*, *validated* and *epm*.
- *original* contain only runs that have been gathered during the optimization-process.
- *validated* may contain original runs, but also data that was not gathered iteratively during the
optimization, but systematically through external validation of interesting configurations.
Important: NO ESTIMATED RUNS IN `validated` RUNHISTORIES!
- *epm* contain runs that are gathered through empirical performance models.
Runhistories are organized as follows:
- each ConfiguratorRun has an *original_runhistory*- and a *combined_runhistory*-attribute
- if available, each ConfiguratorRun's *validated_runhistory* contains
a runhistory with validation-data gathered after the optimization
- *combined_runhistory* always contains as many real runs as possible
CaveFacade contains three runhistories:
- *original_rh*: original runs that have been performed **during optimization**!
- *validated_rh*: runs that have been validated, so they were not part of the original optimization
- *epm_rh*: contains epm-predictions for all incumbents
The analyze()-method performs an analysis and output a report.html.
Arguments
---------
folders: list<strings>
paths to relevant SMAC runs
output_dir: string
output for cave to write results (figures + report)
ta_exec_dir: string
execution directory for target algorithm (to find instance.txt specified in scenario, ..)
file_format: str
what format the rundata is in, options are [SMAC3, SMAC2 and CSV]
validation_method: string
from [validation, epm], how to estimate missing runs
pimp_max_samples: int
passed to PIMP for configuration
fanova_pairwise: bool
whether to calculate pairwise marginals for fanova
use_budgets: bool
if true, individual runs are treated as different budgets. they are not evaluated together, but compared
against each other. runs are expected in ascending budget-size.
seed: int
random seed for analysis (e.g. the random forests)
"""
self.logger = logging.getLogger(self.__module__ + '.' +
self.__class__.__name__)
self.output_dir = output_dir
self.rng = np.random.RandomState(seed)
self.use_budgets = use_budgets
self.ta_exec_dir = ta_exec_dir
self.file_format = file_format
self.validation_format = validation_format
self.validation_method = validation_method
self.pimp_max_samples = pimp_max_samples
self.fanova_pairwise = fanova_pairwise
self.bohb_result = None # only relevant for bohb_result
# Create output_dir if necessary
self.logger.info("Saving results to '%s'", self.output_dir)
if not os.path.exists(output_dir):
self.logger.debug("Output-dir '%s' does not exist, creating",
self.output_dir)
os.makedirs(output_dir)
if file_format == 'BOHB':
if len(folders) != 1:
raise ValueError(
"For file format BOHB you can only specify one folder.")
self.bohb_result, folders = HpBandSter2SMAC().convert(folders[0])
# Save all relevant configurator-runs in a list
self.logger.debug("Folders: %s; ta-exec-dirs: %s", str(folders),
str(ta_exec_dir))
self.runs = []
if len(ta_exec_dir) < len(folders):
for i in range(len(folders) - len(ta_exec_dir)):
ta_exec_dir.append(ta_exec_dir[0])
for ta_exec_dir, folder in zip(ta_exec_dir, folders):
try:
self.logger.debug("Collecting data from %s.", folder)
self.runs.append(
ConfiguratorRun(folder,
ta_exec_dir,
file_format=file_format,
validation_format=validation_format))
except Exception as err:
self.logger.warning(
"Folder %s could with ta_exec_dir %s not be loaded, failed with error message: %s",
folder, ta_exec_dir, err)
self.logger.exception(err)
continue
if not self.runs:
raise ValueError("None of the specified folders could be loaded.")
# Use scenario of first run for general purposes (expecting they are all the same anyway!
self.scenario = self.runs[0].solver.scenario
scenario_sanity_check(self.scenario, self.logger)
self.default = self.scenario.cs.get_default_configuration()
# All runs that have been actually explored during optimization
self.global_original_rh = None
# All original runs + validated runs if available
self.global_validated_rh = None
# All validated runs + EPM-estimated for def and inc on all insts
self.global_epm_rh = None
self.pimp = None
self.model = None
if use_budgets:
self._init_helper_budgets()
else:
self._init_helper_no_budgets()
self.analyzer = Analyzer(self.default,
self.incumbent,
self.scenario,
self.output_dir,
pimp_max_samples,
fanova_pairwise,
rng=self.rng)
# Builder for html-website
custom_logo = './custom_logo.png'
if file_format.startswith('SMAC'):
logo_fn = 'SMAC_logo.png'
elif file_format == 'BOHB':
logo_fn = 'BOHB_logo.png'
elif os.path.exists(custom_logo):
logo_fn = custom_logo
else:
logo_fn = 'ml4aad.png'
self.logger.info(
"No suitable logo found. You can use a custom logo simply by having a file called '%s' "
"in the directory from which you run CAVE.", custom_logo)
self.builder = HTMLBuilder(self.output_dir,
"CAVE",
logo_fn=logo_fn,
logo_custom=custom_logo == logo_fn)
self.website = OrderedDict([])
def _init_helper_budgets(self):
self.best_run = self.runs[-1]
self.incumbent = self.best_run.solver.incumbent
def _init_helper_no_budgets(self):
"""No budgets means using global, aggregated runhistories to analyze the Configurator's behaviour.
Also it creates an EPM using all available information, since all runs are "equal".
"""
self.global_original_rh = RunHistory(average_cost)
self.global_validated_rh = RunHistory(average_cost)
self.global_epm_rh = RunHistory(
average_cost) # Save all relevant SMAC-runs in a list
self.logger.debug("Update original rh with all available rhs!")
for run in self.runs:
self.global_original_rh.update(run.original_runhistory,
origin=DataOrigin.INTERNAL)
self.global_validated_rh.update(run.original_runhistory,
origin=DataOrigin.INTERNAL)
if run.validated_runhistory:
self.global_validated_rh.update(
run.validated_runhistory,
origin=DataOrigin.EXTERNAL_SAME_INSTANCES)
self._init_pimp_and_validator(self.global_validated_rh)
# Estimate missing costs for [def, inc1, inc2, ...]
self.validate_default_and_incumbents(self.validation_method,
self.ta_exec_dir)
self.global_epm_rh.update(self.global_validated_rh)
for rh_name, rh in [("original", self.global_original_rh),
("validated", self.global_validated_rh),
("epm", self.global_epm_rh)]:
self.logger.debug(
'Combined number of RunHistory data points for %s runhistory: %d '
'# Configurations: %d. # Configurator runs: %d', rh_name,
len(rh.data), len(rh.get_all_configs()), len(self.runs))
# Sort runs (best first)
self.runs = sorted(
self.runs,
key=lambda run: self.global_epm_rh.get_cost(run.solver.incumbent))
self.best_run = self.runs[0]
self.incumbent = self.pimp.incumbent = self.best_run.solver.incumbent
self.logger.debug("Overall best run: %s, with incumbent: %s",
self.best_run.folder, self.incumbent)
def _init_pimp_and_validator(self, rh, alternative_output_dir=None):
"""Create ParameterImportance-object and use it's trained model for validation and further predictions
We pass validated runhistory, so that the returned model will be based on as much information as possible
Parameters
----------
rh: RunHistory
runhistory used to build EPM
alternative_output_dir: str
e.g. for budgets we want pimp to use an alternative output-dir (subfolders per budget)
"""
self.logger.debug(
"Using '%s' as output for pimp", alternative_output_dir
if alternative_output_dir else self.output_dir)
self.pimp = Importance(
scenario=copy.deepcopy(self.scenario),
runhistory=rh,
incumbent=self.default, # Inject correct incumbent later
parameters_to_evaluate=4,
save_folder=alternative_output_dir
if alternative_output_dir else self.output_dir,
seed=self.rng.randint(1, 100000),
max_sample_size=self.pimp_max_samples,
fANOVA_pairwise=self.fanova_pairwise,
preprocess=False)
self.model = self.pimp.model
# Validator (initialize without trajectory)
self.validator = Validator(self.scenario, None, None)
self.validator.epm = self.model
@timing
def validate_default_and_incumbents(self, method, ta_exec_dir):
"""Validate default and incumbent configurations on all instances possible.
Either use validation (physically execute the target algorithm) or EPM-estimate and update according runhistory
(validation -> self.global_validated_rh; epm -> self.global_epm_rh).
Parameters
----------
method: str
epm or validation
ta_exec_dir: str
path from where the target algorithm can be executed as found in scenario (only used for actual validation)
"""
for run in self.runs:
self.logger.debug("Validating %s using %s!", run.folder, method)
self.validator.traj = run.traj
if method == "validation":
with changedir(ta_exec_dir):
# TODO determine # repetitions
new_rh = self.validator.validate(
'def+inc',
'train+test',
1,
-1,
runhistory=self.global_validated_rh)
self.global_validated_rh.update(new_rh)
elif method == "epm":
# Only do test-instances if features for test-instances are available
instance_mode = 'train+test'
if (any([
i not in self.scenario.feature_dict
for i in self.scenario.test_insts
]) and any([
i in self.scenario.feature_dict
for i in self.scenario.train_insts
])): # noqa
self.logger.debug(
"No features provided for test-instances (but for train!). "
"Cannot validate on \"epm\".")
self.logger.warning(
"Features detected for train-instances, but not for test-instances. This is "
"unintended usage and may lead to errors for some analysis-methods."
)
instance_mode = 'train'
new_rh = self.validator.validate_epm(
'def+inc',
instance_mode,
1,
runhistory=self.global_validated_rh)
self.global_epm_rh.update(new_rh)
else:
raise ValueError("Missing data method illegal (%s)", method)
self.validator.traj = None # Avoid usage-mistakes
@timing
def analyze(self,
performance=True,
cdf=True,
scatter=True,
cfp=True,
cfp_time_slider=False,
cfp_max_plot=-1,
cfp_number_quantiles=10,
param_importance=['forward_selection', 'ablation', 'fanova'],
pimp_sort_table_by: str = "average",
feature_analysis=[
"box_violin", "correlation", "importance", "clustering",
"feature_cdf"
],
parallel_coordinates=True,
cost_over_time=True,
algo_footprint=True):
"""Analyze the available data and build HTML-webpage as dict.
Save webpage in 'self.output_dir/CAVE/report.html'.
Analyzing is performed with the analyzer-instance that is initialized in
the __init__
Parameters
----------
performance: bool
whether to calculate par10-values
cdf: bool
whether to plot cdf
scatter: bool
whether to plot scatter
cfp: bool
whether to perform configuration visualization
cfp_time_slider: bool
whether to include an interactive time-slider in configuration footprint
cfp_max_plot: int
limit number of configurations considered for configuration footprint (-1 -> all configs)
cfp_number_quantiles: int
number of steps over time generated in configuration footprint
param_importance: List[str]
containing methods for parameter importance
pimp_sort_table: str
in what order the parameter-importance overview should be organized
feature_analysis: List[str]
containing methods for feature analysis
parallel_coordinates: bool
whether to plot parallel coordinates
cost_over_time: bool
whether to plot cost over time
algo_footprint: bool
whether to plot algorithm footprints
"""
# Check arguments
for p in param_importance:
if p not in ['forward_selection', 'ablation', 'fanova', 'lpi']:
raise ValueError(
"%s not a valid option for parameter importance!" % p)
for f in feature_analysis:
if f not in [
"box_violin", "correlation", "importance", "clustering",
"feature_cdf"
]:
raise ValueError(
"%s not a valid option for feature analysis!" % f)
# Start analysis
headings = [
"Meta Data", "Best Configuration", "Performance Analysis",
"Configurator's Behavior", "Parameter Importance",
"Feature Analysis"
]
for h in headings:
self.website[h] = OrderedDict()
if self.use_budgets:
# The individual configurator runs are not directory comparable and cannot be aggregated.
# Nevertheless they need to be combined in one comprehensive report and some metrics are to be compared over
# the individual runs.
# if self.file_format == 'BOHB':
# self.website["BOHB Visualization"] = {"figure" : [self.analyzer.bohb_plot(self.bohb_result)]}
# Perform analysis for each run
for run in self.runs:
sub_sec = os.path.basename(run.folder)
# Set paths for each budget individual to avoid path-conflicts
sub_output_dir = os.path.join(self.output_dir, 'content',
sub_sec)
os.makedirs(sub_output_dir, exist_ok=True)
self.analyzer = Analyzer(run.default,
run.incumbent,
self.scenario,
sub_output_dir,
self.pimp_max_samples,
self.fanova_pairwise,
rng=self.rng)
# Set runhistories
self.global_original_rh = run.original_runhistory
self.global_validated_rh = run.combined_runhistory
self.global_epm_rh = RunHistory(average_cost)
# Train epm and stuff
self._init_pimp_and_validator(
run.combined_runhistory,
alternative_output_dir=sub_output_dir)
self.validate_default_and_incumbents(self.validation_method,
run.ta_exec_dir)
self.pimp.incumbent = run.incumbent
self.incumbent = run.incumbent
run.epm_rh = self.global_epm_rh
self.best_run = run
# Perform analysis
overview = self.analyzer.create_overview_table(
self.global_original_rh, run, len(self.runs), self.default,
self.incumbent)
self.website["Meta Data"][sub_sec] = {"table": overview}
compare_config_html = compare_configs_to_html(
self.default, self.incumbent)
self.website["Best Configuration"][sub_sec] = {
"table": compare_config_html
}
d = self.website["Performance Analysis"][
sub_sec] = OrderedDict()
self.performance_analysis(d, performance, cdf, scatter,
algo_footprint)
d = self.website["Parameter Importance"][
sub_sec] = OrderedDict()
self.parameter_importance(
d,
ablation='ablation' in param_importance,
fanova='fanova' in param_importance,
forward_selection='forward_selection' in param_importance,
lpi='lpi' in param_importance,
pimp_sort_table_by=pimp_sort_table_by)
d = self.website["Configurator's Behavior"][
sub_sec] = OrderedDict()
self.configurators_behavior(d, cost_over_time, cfp,
cfp_max_plot, cfp_time_slider,
cfp_number_quantiles,
parallel_coordinates)
d = self.website["Feature Analysis"][sub_sec] = OrderedDict()
self.feature_analysis(d,
box_violin='box_violin'
in feature_analysis,
correlation='correlation'
in feature_analysis,
clustering='clustering'
in feature_analysis,
importance='importance'
in feature_analysis)
self.original_runhistory = self.validated_runhistory = self.epm_runhistory = None
else:
overview = self.analyzer.create_overview_table(
self.global_original_rh, self.runs[0], len(self.runs),
self.default, self.incumbent)
self.website["Meta Data"] = {"table": overview}
compare_config_html = compare_configs_to_html(
self.default, self.incumbent)
self.website["Best Configuration"] = {"table": compare_config_html}
self.performance_analysis(self.website["Performance Analysis"],
performance, cdf, scatter,
algo_footprint)
self.parameter_importance(self.website["Parameter Importance"],
ablation='ablation' in param_importance,
fanova='fanova' in param_importance,
forward_selection='forward_selection'
in param_importance,
lpi='lpi' in param_importance,
pimp_sort_table_by=pimp_sort_table_by)
self.configurators_behavior(
self.website["Configurator's Behavior"], cost_over_time, cfp,
cfp_max_plot, cfp_time_slider, cfp_number_quantiles,
parallel_coordinates)
self.feature_analysis(self.website["Feature Analysis"],
box_violin='box_violin' in feature_analysis,
correlation='correlation'
in feature_analysis,
clustering='clustering' in feature_analysis,
importance='importance' in feature_analysis)
self.build_website()
self.logger.info("CAVE finished. Report is located in %s",
os.path.join(self.output_dir, 'report.html'))
def performance_analysis(self, d, performance, cdf, scatter,
algo_footprint):
"""Generate performance analysis.
Parameters
----------
d: dictionary
dictionary to add entries to
performance, cdf, scatter, algo_footprint: bool
what analysis-methods to perform
"""
if performance:
instances = [
i for i in self.scenario.train_insts + self.scenario.test_insts
if i
]
oracle = self.analyzer.get_oracle(instances,
self.global_validated_rh)
performance_table = self.analyzer.create_performance_table(
self.default, self.incumbent, self.global_epm_rh, oracle)
d["Performance Table"] = {"table": performance_table}
if cdf:
cdf_paths = self.analyzer.plot_cdf_compare(self.default,
self.incumbent,
self.global_epm_rh)
if cdf_paths:
d["empirical Cumulative Distribution Function (eCDF)"] = {
"figure": cdf_paths
}
if scatter:
scatter_paths = self.analyzer.plot_scatter(self.default,
self.incumbent,
self.global_epm_rh)
if scatter_paths:
d["Scatterplot"] = {"figure": scatter_paths}
self.build_website()
if algo_footprint and self.scenario.feature_dict:
algorithms = [(self.default, "default"),
(self.incumbent, "incumbent")]
algo_footprint_plots = self.analyzer.plot_algorithm_footprint(
self.global_epm_rh, algorithms)
d["Algorithm Footprints"] = OrderedDict()
# Interactive bokeh-plot
script, div = algo_footprint_plots[0]
d["Algorithm Footprints"]["Interactive Algorithm Footprint"] = {
"bokeh": (script, div)
}
p_3d = algo_footprint_plots[1]
for plots in p_3d:
header = os.path.splitext(os.path.split(plots[0])[1])[0][10:-2]
header = header[0].upper() + header[1:].replace('_', ' ')
d["Algorithm Footprints"][header] = {"figure_x2": plots}
self.build_website()
def configurators_behavior(self,
d,
cost_over_time=False,
cfp=False,
cfp_max_plot=-1,
cfp_time_slider=False,
cfp_number_quantiles=1,
parallel_coordinates=False):
if cost_over_time:
cost_over_time_script = self.analyzer.plot_cost_over_time(
self.global_validated_rh, self.runs, self.validator)
d["Cost Over Time"] = {"bokeh": cost_over_time_script}
self.build_website()
if cfp: # Configurator Footprint
runs = [self.best_run] if self.use_budgets else self.runs
res = self.analyzer.plot_configurator_footprint(
self.scenario,
runs,
self.global_original_rh,
max_confs=cfp_max_plot,
time_slider=(cfp_time_slider and (cfp_number_quantiles > 1)),
num_quantiles=cfp_number_quantiles)
bokeh_components, cfp_paths = res
if cfp_number_quantiles == 1: # Only one plot, no need for "Static"-field
d["Configurator Footprint"] = {"bokeh": (bokeh_components)}
else:
d["Configurator Footprint"] = OrderedDict()
d["Configurator Footprint"]["Interactive"] = {
"bokeh": (bokeh_components)
}
if all([True for p in cfp_paths if os.path.exists(p)
]): # If the plots were actually generated
d["Configurator Footprint"]["Static"] = {
"figure": cfp_paths
}
else:
d["Configurator Footprint"]["Static"] = {
"else":
"This plot is missing. Maybe it was not generated? "
"Check if you installed selenium and phantomjs "
"correctly to activate bokeh-exports. "
"(https://automl.github.io/CAVE/stable/faq.html)"
}
self.build_website()
if parallel_coordinates:
# Should be after parameter importance, if performed.
n_params = 6
parallel_path = self.analyzer.plot_parallel_coordinates(
self.global_original_rh, self.global_validated_rh,
self.validator, n_params)
if parallel_path:
d["Parallel Coordinates"] = {"figure": parallel_path}
self.build_website()
def parameter_importance(self,
d,
ablation=False,
fanova=False,
forward_selection=False,
lpi=False,
pimp_sort_table_by='average'):
"""Perform the specified parameter importance procedures. """
sum_ = 0
if fanova:
sum_ += 1
self.logger.info("fANOVA...")
d["fANOVA"] = OrderedDict()
try:
table, plots, pair_plots = self.analyzer.fanova(
self.pimp, self.incumbent)
d["fANOVA"]["Importance"] = {"table": table}
# Insert plots (the received plots is a dict, mapping param -> path)
d["fANOVA"]["Marginals"] = OrderedDict()
for param, plot in plots.items():
d["fANOVA"]["Marginals"][param] = {"figure": plot}
if pair_plots:
d["fANOVA"]["Pairwise Marginals"] = OrderedDict()
for param, plot in pair_plots.items():
d["fANOVA"]["Pairwise Marginals"][param] = {
"figure": plot
}
except RuntimeError as e:
err = "Encountered error '%s' in fANOVA, this can e.g. happen with too few data-points." % e
self.logger.exception(err)
d["fANOVA"] = {
"else":
err + " Check 'debug/debug.log' for more information."
}
self.build_website()
if ablation:
sum_ += 1
self.logger.info("Ablation...")
self.analyzer.parameter_importance(self.pimp, "ablation",
self.incumbent,
self.analyzer.output_dir)
ablationpercentage_path = os.path.join(self.analyzer.output_dir,
"ablationpercentage.png")
ablationperformance_path = os.path.join(self.analyzer.output_dir,
"ablationperformance.png")
d["Ablation"] = {
"figure": [ablationpercentage_path, ablationperformance_path]
}
self.build_website()
if forward_selection:
sum_ += 1
self.logger.info("Forward Selection...")
self.analyzer.parameter_importance(self.pimp, "forward-selection",
self.incumbent,
self.analyzer.output_dir)
f_s_barplot_path = os.path.join(self.analyzer.output_dir,
"forward-selection-barplot.png")
f_s_chng_path = os.path.join(self.analyzer.output_dir,
"forward-selection-chng.png")
d["Forward Selection"] = {
"figure": [f_s_barplot_path, f_s_chng_path]
}
self.build_website()
if lpi:
sum_ += 1
self.logger.info("Local EPM-predictions around incumbent...")
plots = self.analyzer.local_epm_plots(self.pimp)
d["Local Parameter Importance (LPI)"] = OrderedDict()
for param, plot in plots.items():
d["Local Parameter Importance (LPI)"][param] = {"figure": plot}
self.build_website()
if sum_ >= 2:
out_fn = os.path.join(self.output_dir, 'pimp.tex')
self.logger.info('Creating pimp latex table at %s' % out_fn)
self.pimp.table_for_comparison(self.analyzer.evaluators,
out_fn,
style='latex')
table = self.analyzer.importance_table(pimp_sort_table_by)
d["Importance Table"] = {
"table":
table,
"tooltip":
"Parameters are sorted by {}. Note, that the values are not "
"directly comparable, since the different techniques "
"provide different metrics (see respective tooltips "
"for details on the differences).".format(pimp_sort_table_by)
}
d.move_to_end("Importance Table", last=False)
self.build_website()
def feature_analysis(self,
d,
box_violin=False,
correlation=False,
clustering=False,
importance=False):
if not self.scenario.feature_dict:
self.logger.error(
"No features available. Skipping feature analysis.")
return
feat_fn = self.scenario.feature_fn
if not self.scenario.feature_names:
self.logger.debug(
"`scenario.feature_names` is not set. Loading from '%s'",
feat_fn)
with changedir(self.ta_exec_dir if self.ta_exec_dir else '.'):
if not feat_fn or not os.path.exists(feat_fn):
self.logger.warning(
"Feature names are missing. Either provide valid feature_file in scenario "
"(currently %s) or set `scenario.feature_names` manually."
% feat_fn)
self.logger.error("Skipping Feature Analysis.")
return
else:
# Feature names are contained in feature-file and retrieved
feat_names = InputReader().read_instance_features_file(
feat_fn)[0]
else:
feat_names = copy.deepcopy(self.scenario.feature_names)
# feature importance using forward selection
if importance:
d["Feature Importance"] = OrderedDict()
imp, plots = self.analyzer.feature_importance(self.pimp)
imp = DataFrame(data=list(imp.values()),
index=list(imp.keys()),
columns=["Error"])
imp = imp.to_html() # this is a table with the values in html
d["Feature Importance"]["Table"] = {"table": imp}
for p in plots:
name = os.path.splitext(os.path.basename(p))[0]
d["Feature Importance"][name] = {"figure": p}
# box and violin plots
if box_violin:
name_plots = self.analyzer.feature_analysis(
'box_violin', feat_names)
d["Violin and Box Plots"] = OrderedDict()
for plot_tuple in name_plots:
key = "%s" % (plot_tuple[0])
d["Violin and Box Plots"][key] = {"figure": plot_tuple[1]}
# correlation plot
if correlation:
correlation_plot = self.analyzer.feature_analysis(
'correlation', feat_names)
if correlation_plot:
d["Correlation"] = {"figure": correlation_plot}
# cluster instances in feature space
if clustering:
cluster_plot = self.analyzer.feature_analysis(
'clustering', feat_names)
d["Clustering"] = {"figure": cluster_plot}
self.build_website()
def build_website(self):
self.builder.generate_html(self.website)
class PIMP:
def __init__(self,
scenario: Scenario,
smac: Union[SMAC, None] = None,
mode: str = 'all',
X: Union[None, List[list], np.ndarray] = None,
y: Union[None, List[list], np.ndarray] = None,
numParams: int = -1,
impute: bool = False,
seed: int = 12345,
run: bool = False,
max_sample_size: int = -1):
"""
Interface to be used with SMAC or with X and y matrices.
:param scenario: The scenario object, that knows the configuration space.
:param smac: The smac object that keeps all the run-data
:param mode: The mode with which to run PIMP [ablation, fanova, all, forward-selection]
:param X: Numpy Array that contains parameter arrays
:param y: Numpy array that contains the corresponding performance values
:param numParams: The number of parameters to evaluate
:param impute: Flag to decide if censored data gets imputed or not
:param seed: The random seed
:param run: Flag to immediately compute the importance values after this setup or not.
"""
self.scenario = scenario
self.imp = None
self.mode = mode
self.save_folder = scenario.output_dir
if smac is not None:
self.imp = Importance(scenario=scenario,
runhistory=smac.runhistory,
incumbent=smac.solver.incumbent,
seed=seed,
parameters_to_evaluate=numParams,
save_folder='PIMP',
impute_censored=impute,
max_sample_size=max_sample_size)
elif X is not None and y is not None:
X = np.array(X)
y = np.array(y)
runHist = RunHistory(average_cost)
if X.shape[0] != y.shape[0]:
raise Exception('Number of samples in X and y dont match!')
n_params = len(scenario.cs.get_hyperparameters())
feats = None
if X.shape[1] > n_params:
feats = X[:, n_params:]
assert feats.shape[1] == scenario.feature_array.shape[1]
X = X[:, :n_params]
for p in range(
X.shape[1]): # Normalize the data to fit into [0, 1]
_min, _max = np.min(X[:, p]), np.max(X[:, p])
if _min < 0. or 1 < _max: # if it is not already normalized
for id, v in enumerate(X[:, p]):
X[id, p] = (v - _min) / (_max - _min)
# Add everything to a runhistory such that PIMP can work with it
for x, feat, y_val in zip(X, feats if feats is not None else X, y):
id = None
for inst in scenario.feature_dict: # determine on which instance a configuration was run
if np.all(scenario.feature_dict[inst] == feat):
id = inst
break
runHist.add(Configuration(scenario.cs, vector=x), y_val, 0,
StatusType.SUCCESS, id)
self.X = X
self.y = y
best_ = None # Determine incumbent according to the best mean cost in the runhistory
for config in runHist.config_ids:
inst_seed_pairs = runHist.get_runs_for_config(config)
all_ = []
for inst, seed in inst_seed_pairs:
rk = RunKey(runHist.config_ids[config], inst, seed)
all_.append(runHist.data[rk].cost)
mean = np.mean(all_)
if best_ is None or best_[0] > mean:
best_ = (mean, config)
incumbent = best_[1]
self.imp = Importance(scenario=scenario,
runhistory=runHist,
seed=seed,
parameters_to_evaluate=numParams,
save_folder='PIMP',
impute_censored=impute,
incumbent=incumbent)
else:
raise Exception(
'Neither X and y matrices nor a SMAC object were specified to compute the importance '
'values from!')
if run:
return self.compute_importances()
def compute_importances(self, order=3):
result = self.imp.evaluate_scenario(self.mode, sort_by=order)
return result
def plot_results(self,
result: Union[List[Dict[str, float]], Dict[str, float]],
save_table: bool = True,
show=False):
save_folder = self.save_folder
if self.mode == 'all':
with open(
os.path.join(save_folder,
'pimp_values_%s.json' % self.mode),
'w') as out_file:
json.dump(result[0],
out_file,
sort_keys=True,
indent=4,
separators=(',', ': '))
self.imp.plot_results(list(
map(lambda x: os.path.join(save_folder, x.name.lower()),
result[1])),
result[1],
show=show)
if save_table:
self.imp.table_for_comparison(
evaluators=result[1],
name=os.path.join(save_folder,
'pimp_table_%s.tex' % self.mode),
style='latex')
else:
self.imp.table_for_comparison(evaluators=result[1],
style='cmd')
else:
with open(
os.path.join(save_folder,
'pimp_values_%s.json' % self.mode),
'w') as out_file:
json.dump(result,
out_file,
sort_keys=True,
indent=4,
separators=(',', ': '))
self.imp.plot_results(name=os.path.join(save_folder, self.mode),
show=show)
def cmd_line_call():
"""
Main Parameter importance script.
"""
cmd_reader = CMDs()
args, misc_ = cmd_reader.read_cmd() # read cmd args
logging.basicConfig(level=args.verbose_level)
ts = time.time()
ts = datetime.datetime.fromtimestamp(ts).strftime('%Y_%m_%d_%H:%M:%S')
if not args.out_folder:
save_folder = 'PIMP_%s_%s' % (args.modus, ts)
else:
if os.path.exists(os.path.abspath(args.out_folder)) or os.path.exists(
os.path.abspath(args.out_folder + '_%s' % args.modus)):
save_folder = args.out_folder + '_%s_%s' % (args.modus, ts)
else:
save_folder = args.out_folder + '_%s' % args.modus
importance = Importance(
scenario_file=args.scenario_file,
runhistory_file=args.history,
parameters_to_evaluate=args.num_params,
traj_file=args.trajectory,
seed=args.seed,
save_folder=save_folder,
impute_censored=args.impute,
max_sample_size=args.max_sample_size) # create importance object
with open(os.path.join(save_folder, 'pimp_args.json'), 'w') as out_file:
json.dump(args.__dict__,
out_file,
sort_keys=True,
indent=4,
separators=(',', ': '))
result = importance.evaluate_scenario(args.modus, sort_by=args.order)
if args.modus == 'all':
with open(
os.path.join(save_folder, 'pimp_values_%s.json' % args.modus),
'w') as out_file:
json.dump(result[0],
out_file,
sort_keys=True,
indent=4,
separators=(',', ': '))
importance.plot_results(list(
map(lambda x: os.path.join(save_folder, x.name.lower()),
result[1])),
result[1],
show=False)
if args.table:
importance.table_for_comparison(
evaluators=result[1],
name=os.path.join(save_folder,
'pimp_table_%s.tex' % args.modus),
style='latex')
else:
importance.table_for_comparison(evaluators=result[1], style='cmd')
else:
with open(
os.path.join(save_folder, 'pimp_values_%s.json' % args.modus),
'w') as out_file:
json.dump(result,
out_file,
sort_keys=True,
indent=4,
separators=(',', ': '))
importance.plot_results(name=os.path.join(save_folder, args.modus),
show=False)
from pimp.importance.importance import Importance
from pimp.utils.io.cmd_reader import CMDs
__author__ = "Andre Biedenkapp"
__copyright__ = "Copyright 2016, ML4AAD"
__license__ = "3-clause BSD"
__maintainer__ = "Andre Biedenkapp"
__email__ = "*****@*****.**"
if __name__ == '__main__':
"""
Main Parameter importance script.
"""
cmd_reader = CMDs()
args, misc_ = cmd_reader.read_cmd() # read cmd args
logging.basicConfig(level=args.verbose_level)
importance = Importance(args.scenario_file,
args.history,
parameters_to_evaluate=args.num_params,
traj_file=args.trajectory,
seed=args.seed) # create importance object
importance_value_dict = importance.evaluate_scenario(args.modus)
ts = time.time()
ts = datetime.datetime.fromtimestamp(ts).strftime('%Y_%m_%d_%H:%M:%S')
with open('pimp_values_%s_%s.json' % (args.modus, ts), 'w') as out_file:
json.dump(importance_value_dict, out_file)
importance.plot_results(name=args.modus)
