def test_convert_restrictions(self):
# This is a smoke test to make sure that the int/float values in the
# greater or smaller statements are converted to the right type when
# reading them
s = """x1 real [0,1] [0]
x2 real [0,1] [0]
x3 real [0,1] [0]
x4 integer [0,2] [0]
x5 real [0,1] [0]
x6 ordinal {cold, luke-warm, hot} [cold]
x1 | x2 > 0.5
x3 | x4 > 1 && x4 == 2 && x4 in {2}
x5 | x6 > luke-warm"""
pcs_new.read(s.split('\n'))
def test_convert_restrictions(self):
# This is a smoke test to make sure that the int/float values in the
# greater or smaller statements are converted to the right type when
# reading them
s = """x1 real [0,1] [0]
x2 real [0,1] [0]
x3 real [0,1] [0]
x4 integer [0,2] [0]
x5 real [0,1] [0]
x6 ordinal {cold, luke-warm, hot} [cold]
x1 | x2 > 0.5
x3 | x4 > 1 && x4 == 2 && x4 in {2}
x5 | x6 > luke-warm"""
pcs_new.read(s.split('\n'))
def run_test(self):
try:
with open(configuration_space_path) as fh:
cs = pcs_parser.read(fh)
except:
with open(configuration_space_path) as fh:
cs = pcs_new_parser.read(fh)
default = cs.get_default_configuration()
cs._check_configuration_rigorous(default)
for i in range(10):
neighborhood = ConfigSpace.util.get_one_exchange_neighbourhood(
default, seed=i)
for shuffle, n in enumerate(neighborhood):
n.is_valid_configuration()
cs._check_configuration_rigorous(n)
if shuffle == 10:
break
# Sample a little bit
for i in range(10):
cs.seed(i)
configurations = cs.sample_configuration(size=5)
for j, c in enumerate(configurations):
c.is_valid_configuration()
cs._check_configuration_rigorous(c)
neighborhood = ConfigSpace.util.get_one_exchange_neighbourhood(
c, seed=i)
for shuffle, n in enumerate(neighborhood):
n.is_valid_configuration()
cs._check_configuration_rigorous(n)
if shuffle == 20:
break
def run_test(self):
try:
with open(configuration_space_path) as fh:
cs = pcs_parser.read(fh)
except Exception:
with open(configuration_space_path) as fh:
cs = pcs_new_parser.read(fh)
default = cs.get_default_configuration()
cs._check_configuration_rigorous(default)
for i in range(10):
neighborhood = ConfigSpace.util.get_one_exchange_neighbourhood(
default, seed=i)
for shuffle, n in enumerate(neighborhood):
n.is_valid_configuration()
cs._check_configuration_rigorous(n)
if shuffle == 10:
break
# Sample a little bit
for i in range(10):
cs.seed(i)
configurations = cs.sample_configuration(size=5)
for j, c in enumerate(configurations):
c.is_valid_configuration()
cs._check_configuration_rigorous(c)
neighborhood = ConfigSpace.util.get_one_exchange_neighbourhood(
c, seed=i)
for shuffle, n in enumerate(neighborhood):
n.is_valid_configuration()
cs._check_configuration_rigorous(n)
if shuffle == 20:
break
def test_read_new_configuration_space_complex_conditionals(self):
classi = OrdinalHyperparameter("classi", ["random_forest", "extra_trees", "k_nearest_neighbors", "something"])
knn_weights = CategoricalHyperparameter("knn_weights", ["uniform", "distance"])
weather = OrdinalHyperparameter("weather", ["sunny", "rainy", "cloudy", "snowing"])
temperature = CategoricalHyperparameter("temperature", ["high", "low"])
rain = CategoricalHyperparameter("rain", ["yes", "no"])
gloves = OrdinalHyperparameter("gloves", ["none", "yarn", "leather", "gortex"])
heur1 = CategoricalHyperparameter("heur1", ["off", "on"])
heur2 = CategoricalHyperparameter("heur2", ["off", "on"])
heur_order = CategoricalHyperparameter("heur_order", ["heur1then2", "heur2then1"])
gloves_condition = OrConjunction(EqualsCondition(gloves, rain, "yes"),
EqualsCondition(gloves, temperature, "low"))
heur_condition = AndConjunction(EqualsCondition(heur_order, heur1, "on"),
EqualsCondition(heur_order, heur2, "on"))
and_conjunction = AndConjunction(NotEqualsCondition(knn_weights, classi, "extra_trees"),
EqualsCondition(knn_weights, classi, "random_forest"))
Cl_condition = OrConjunction(EqualsCondition(knn_weights, classi, "k_nearest_neighbors"),
and_conjunction,
EqualsCondition(knn_weights, classi, "something"))
and1 = AndConjunction(EqualsCondition(temperature, weather, "rainy"),
EqualsCondition(temperature, weather, "cloudy"))
and2 = AndConjunction(EqualsCondition(temperature, weather, "sunny"),
NotEqualsCondition(temperature, weather, "snowing"))
another_condition = OrConjunction(and1, and2)
complex_conditional_space = ConfigurationSpace()
complex_conditional_space.add_hyperparameter(classi)
complex_conditional_space.add_hyperparameter(knn_weights)
complex_conditional_space.add_hyperparameter(weather)
complex_conditional_space.add_hyperparameter(temperature)
complex_conditional_space.add_hyperparameter(rain)
complex_conditional_space.add_hyperparameter(gloves)
complex_conditional_space.add_hyperparameter(heur1)
complex_conditional_space.add_hyperparameter(heur2)
complex_conditional_space.add_hyperparameter(heur_order)
complex_conditional_space.add_condition(gloves_condition)
complex_conditional_space.add_condition(heur_condition)
complex_conditional_space.add_condition(Cl_condition)
complex_conditional_space.add_condition(another_condition)
complex_cs = list()
complex_cs.append("classi ordinal {random_forest,extra_trees,k_nearest_neighbors, something} [random_forest]")
complex_cs.append("knn_weights categorical {uniform, distance} [uniform]")
complex_cs.append("weather ordinal {sunny, rainy, cloudy, snowing} [sunny]")
complex_cs.append("temperature categorical {high, low} [high]")
complex_cs.append("rain categorical { yes, no } [yes]")
complex_cs.append("gloves ordinal { none, yarn, leather, gortex } [none]")
complex_cs.append("heur1 categorical { off, on } [off]")
complex_cs.append("heur2 categorical { off, on } [off]")
complex_cs.append("heur_order categorical { heur1then2, heur2then1 } [heur1then2]")
complex_cs.append("gloves | rain == yes || temperature == low")
complex_cs.append("heur_order | heur1 == on && heur2 == on")
complex_cs.append("knn_weights | classi == k_nearest_neighbors || "
"classi != extra_trees && classi == random_forest || classi == something")
complex_cs.append("temperature | weather == rainy && weather == cloudy || "
"weather == sunny && weather != snowing")
cs_new = pcs_new.read(complex_cs)
self.assertEqual(cs_new, complex_conditional_space)
def test_read_new_configuration_space_forbidden(self):
cs_with_forbidden = ConfigurationSpace()
int_hp = UniformIntegerHyperparameter('int_hp', 0, 50, 30)
float_hp = UniformFloatHyperparameter('float_hp', 0., 50., 30.)
cat_hp_str = CategoricalHyperparameter('cat_hp_str', ['a', 'b', 'c'], 'b')
ord_hp_str = OrdinalHyperparameter('ord_hp_str', ['a', 'b', 'c'], 'b')
cs_with_forbidden.add_hyperparameters([int_hp, float_hp, cat_hp_str, ord_hp_str])
int_hp_forbidden = ForbiddenAndConjunction(ForbiddenEqualsClause(int_hp, 1))
float_hp_forbidden_1 = ForbiddenEqualsClause(float_hp, 1.0)
float_hp_forbidden_2 = ForbiddenEqualsClause(float_hp, 2.0)
float_hp_forbidden = ForbiddenAndConjunction(float_hp_forbidden_1, float_hp_forbidden_2)
cat_hp_str_forbidden = ForbiddenAndConjunction(ForbiddenEqualsClause(cat_hp_str, 'a'))
ord_hp_float_forbidden = ForbiddenAndConjunction(ForbiddenEqualsClause(ord_hp_str, 'a'))
cs_with_forbidden.add_forbidden_clauses([int_hp_forbidden, float_hp_forbidden,
cat_hp_str_forbidden, ord_hp_float_forbidden])
complex_cs = list()
complex_cs.append("int_hp integer [0,50] [30]")
complex_cs.append("float_hp real [0.0, 50.0] [30.0]")
complex_cs.append("cat_hp_str categorical {a, b, c} [b]")
complex_cs.append("ord_hp_str ordinal {a, b, c} [b]")
complex_cs.append("# Forbiddens:")
complex_cs.append("{int_hp=1}")
complex_cs.append("{float_hp=1.0, float_hp=2.0}")
complex_cs.append("{cat_hp_str=a}")
complex_cs.append("{ord_hp_str=a}")
cs_new = pcs_new.read(complex_cs)
self.assertEqual(cs_new, cs_with_forbidden)
def load_configspace(self, folder):
"""Will try to load the configspace. If it's a pcs-file, backup_cs will be a list containing all possible
combinations of interpretation for Categoricals. If this issue will be fixed, we can drop this procedure."""
cs_fn_json = os.path.join(folder, 'configspace.json')
cs_fn_pcs = os.path.join(folder, 'configspace.pcs')
if os.path.exists(cs_fn_json):
with open(cs_fn_json, 'r') as fh:
cs = pcs_json.read(fh.read())
backup_cs = []
self.logger.debug(
"Detected and loaded \"%s\". No backup-cs necessary",
cs_fn_json)
elif os.path.exists(cs_fn_pcs):
with open(cs_fn_pcs, 'r') as fh:
cs = pcs_new.read(fh.readlines())
# Create alternative interpretations
categoricals = [
hp for hp in cs.get_hyperparameters()
if isinstance(hp, CategoricalHyperparameter)
]
non_categoricals = [
hp for hp in cs.get_hyperparameters()
if not isinstance(hp, CategoricalHyperparameter)
]
def _get_interpretations(choices):
result = []
if set(choices) == {"True", "False"}:
result.append([True, False])
if all([c.isdigit() for c in choices]):
result.append([int(c) for c in choices])
result.append(choices)
return result
choices_per_cat = [
_get_interpretations(hp.choices) for hp in categoricals
]
combinations = itertools.product(*choices_per_cat)
self.logger.debug(combinations)
backup_cs = []
for combi in combinations:
bcs = ConfigurationSpace()
for hp in non_categoricals:
bcs.add_hyperparameter(hp)
for name, choices in zip([hp.name for hp in categoricals],
combi):
bcs.add_hyperparameter(
CategoricalHyperparameter(name, choices))
bcs.add_conditions(cs.get_conditions())
backup_cs.append(bcs)
self.logger.debug("Sampled %d interpretations of \"%s\"",
len(backup_cs), cs_fn_pcs)
self.logger.debug(choices_per_cat)
else:
raise ValueError("Missing pcs-file at '%s.[pcs|json]'!" %
os.path.join(folder, 'configspace'))
return cs, backup_cs
def test_write_restrictions(self):
s = "c integer [0, 2] [0]\n" + \
"d ordinal {cold, luke-warm, hot} [cold]\n" + \
"e real [0.0, 1.0] [0.0]\n" + \
"b real [0.0, 1.0] [0.0]\n" + \
"a real [0.0, 1.0] [0.0]\n" + \
"\n" + \
"b | d in {luke-warm, hot} || c > 1\n" + \
"a | b == 0.5 && e > 0.5"
a = pcs_new.read(s.split('\n'))
out = pcs_new.write(a)
self.assertEqual(out, s)
def test_write_restrictions(self):
s = "c integer [0, 2] [0]\n" + \
"d ordinal {cold, luke-warm, hot} [cold]\n" + \
"e real [0.0, 1.0] [0.0]\n" + \
"b real [0.0, 1.0] [0.0]\n" + \
"a real [0.0, 1.0] [0.0]\n" + \
"\n" + \
"b | d in {luke-warm, hot} || c > 1\n" + \
"a | b == 0.5 && e > 0.5"
a = pcs_new.read(s.split('\n'))
out = pcs_new.write(a)
self.assertEqual(out, s)
def test_read_new_configuration_space_easy(self):
expected = StringIO()
expected.write('# This is a \n')
expected.write(' # This is a comment with a leading whitespace ### ffds \n')
expected.write('\n')
expected.write('float_a real [-1.23, 6.45] [2.61] # bla\n')
expected.write('e_float_a real [.5E-2, 4.5e+06] [2250000.0025]\n')
expected.write('int_a integer [-1, 6] [2]\n')
expected.write('log_a real [4e-1, 6.45] [1.6062378404]log\n')
expected.write('int_log_a integer [1, 6] [2]log\n')
expected.write('cat_a categorical {a,"b",c,d} [a]\n')
expected.write(r'@.:;/\?!$%&_-<>*+1234567890 categorical {"const"} ["const"]\n')
expected.seek(0)
cs = pcs_new.read(expected)
self.assertEqual(cs, easy_space)
def test_read_new_configuration_space_easy(self):
expected = StringIO()
expected.write('# This is a \n')
expected.write(' # This is a comment with a leading whitespace ### ffds \n')
expected.write('\n')
expected.write('float_a real [-1.23, 6.45] [2.61] # bla\n')
expected.write('e_float_a real [.5E-2, 4.5e+06] [2250000.0025]\n')
expected.write('int_a integer [-1, 6] [2]\n')
expected.write('log_a real [4e-1, 6.45] [1.6062378404]log\n')
expected.write('int_log_a integer [1, 6] [2]log\n')
expected.write('cat_a categorical {a,"b",c,d} [a]\n')
expected.write(r'@.:;/\?!$%&_-<>*+1234567890 categorical {"const"} ["const"]\n')
expected.seek(0)
cs = pcs_new.read(expected)
self.assertEqual(cs, easy_space)
def test_read_new_configuration_space_conditional(self):
# More complex search space as string array
complex_cs = list()
complex_cs.append("preprocessing categorical {None, pca} [None]")
complex_cs.append("classifier categorical {svm, nn} [svm]")
complex_cs.append("kernel categorical {rbf, poly, sigmoid} [rbf]")
complex_cs.append("C real [0.03125, 32768] [32]log")
complex_cs.append("neurons integer [16, 1024] [520] # Should be Q16")
complex_cs.append("lr real [0.0001, 1.0] [0.50005]")
complex_cs.append("degree integer [1, 5] [3]")
complex_cs.append("gamma real [0.000030518, 8] [0.0156251079996]log")
complex_cs.append("C | classifier in {svm}")
complex_cs.append("kernel | classifier in {svm}")
complex_cs.append("lr | classifier in {nn}")
complex_cs.append("neurons | classifier in {nn}")
complex_cs.append("degree | kernel in {poly, sigmoid}")
complex_cs.append("gamma | kernel in {rbf}")
cs_new = pcs_new.read(complex_cs)
self.assertEqual(cs_new, conditional_space)
# same in older version
complex_cs_old = list()
complex_cs_old.append("preprocessing {None, pca} [None]")
complex_cs_old.append("classifier {svm, nn} [svm]")
complex_cs_old.append("kernel {rbf, poly, sigmoid} [rbf]")
complex_cs_old.append("C [0.03125, 32768] [32]l")
complex_cs_old.append("neurons [16, 1024] [520]i # Should be Q16")
complex_cs_old.append("lr [0.0001, 1.0] [0.50005]")
complex_cs_old.append("degree [1, 5] [3]i")
complex_cs_old.append("gamma [0.000030518, 8] [0.0156251079996]l")
complex_cs_old.append("C | classifier in {svm}")
complex_cs_old.append("kernel | classifier in {svm}")
complex_cs_old.append("lr | classifier in {nn}")
complex_cs_old.append("neurons | classifier in {nn}")
complex_cs_old.append("degree | kernel in {poly, sigmoid}")
complex_cs_old.append("gamma | kernel in {rbf}")
cs_old = pcs.read(complex_cs_old)
self.assertEqual(cs_old, cs_new)
def test_read_new_configuration_space_conditional(self):
# More complex search space as string array
complex_cs = list()
complex_cs.append("preprocessing categorical {None, pca} [None]")
complex_cs.append("classifier categorical {svm, nn} [svm]")
complex_cs.append("kernel categorical {rbf, poly, sigmoid} [rbf]")
complex_cs.append("C real [0.03125, 32768] [32]log")
complex_cs.append("neurons integer [16, 1024] [520] # Should be Q16")
complex_cs.append("lr real [0.0001, 1.0] [0.50005]")
complex_cs.append("degree integer [1, 5] [3]")
complex_cs.append("gamma real [0.000030518, 8] [0.0156251079996]log")
complex_cs.append("C | classifier in {svm}")
complex_cs.append("kernel | classifier in {svm}")
complex_cs.append("lr | classifier in {nn}")
complex_cs.append("neurons | classifier in {nn}")
complex_cs.append("degree | kernel in {poly, sigmoid}")
complex_cs.append("gamma | kernel in {rbf}")
cs_new = pcs_new.read(complex_cs)
self.assertEqual(cs_new, conditional_space)
# same in older version
complex_cs_old = list()
complex_cs_old.append("preprocessing {None, pca} [None]")
complex_cs_old.append("classifier {svm, nn} [svm]")
complex_cs_old.append("kernel {rbf, poly, sigmoid} [rbf]")
complex_cs_old.append("C [0.03125, 32768] [32]l")
complex_cs_old.append("neurons [16, 1024] [520]i # Should be Q16")
complex_cs_old.append("lr [0.0001, 1.0] [0.50005]")
complex_cs_old.append("degree [1, 5] [3]i")
complex_cs_old.append("gamma [0.000030518, 8] [0.0156251079996]l")
complex_cs_old.append("C | classifier in {svm}")
complex_cs_old.append("kernel | classifier in {svm}")
complex_cs_old.append("lr | classifier in {nn}")
complex_cs_old.append("neurons | classifier in {nn}")
complex_cs_old.append("degree | kernel in {poly, sigmoid}")
complex_cs_old.append("gamma | kernel in {rbf}")
cs_old = pcs.read(complex_cs_old)
self.assertEqual(cs_old, cs_new)
def convert(self, folder):
try:
from hpbandster.core.result import Result as HPBResult
from hpbandster.core.result import logged_results_to_HB_result
except ImportError as e:
raise ImportError(
"To analyze BOHB-data, please install hpbandster (e.g. `pip install hpbandster`)"
)
result = logged_results_to_HB_result(folder)
cs_fn = os.path.join(folder, 'configspace.pcs')
if not os.path.exists(cs_fn):
raise ValueError("Missing pcs-file at '%s'!" % cs_fn)
with open(cs_fn, 'r') as fh:
cs = pcs_new.read(fh.readlines())
# Using temporary files for the intermediate smac-result-like format
tmp_dir = tempfile.mkdtemp()
paths = list(self.hpbandster2smac(result, cs, tmp_dir).values())
return result, paths
def load_configspace(path_to_cs_file):
"""
Load configuration space definition
Args:
path_to_cs_file: Path to the file, in which the configuration space is
defined. Must be in format pcs or json
Returns:
ConfigSpace.configuration_space
"""
if path_to_cs_file.endswith('.pcs'):
with open(path_to_cs_file, 'r') as f:
cfg = pcs_new.read(f)
elif path_to_cs_file.endswith('.json'):
with open(path_to_cs_file, 'r') as f:
cfg = json.read(f.read())
else:
raise ImportError('Configuration space definition not understood. File'
' must be in format pcs or json.')
return cfg
def execute(save_folder, runhistory_location, configspace_location, manual_logtransform, use_percentiles, interaction_effect, n_trees, run_limit=None, draw_plots=True):
with open(runhistory_location) as runhistory_file:
runhistory = json.load(runhistory_file)
with open(configspace_location) as configspace_file:
configspace = read(configspace_file)
os.makedirs(save_folder, exist_ok=True)
X = []
y = []
for item in runhistory['data']:
if run_limit is not None and len(X) > run_limit:
break
valid = True
current = []
setup_id = str(item[0][0])
configuration = runhistory['configs'][setup_id]
for param in configspace.get_hyperparameters():
value = configuration[param.name]
if isinstance(param, ConfigSpace.hyperparameters.UniformFloatHyperparameter) and not isinstance(value, float):
valid = False
elif isinstance(param, ConfigSpace.hyperparameters.UniformIntegerHyperparameter) and not isinstance(value, int):
valid = False
if isinstance(param, ConfigSpace.hyperparameters.CategoricalHyperparameter):
value = param.choices.index(value)
elif param.log and manual_logtransform:
value = np.log(value)
current.append(value)
if valid:
X.append(current)
y.append(item[1][0])
else:
print('Illegal configuration', current)
X = np.array(X)
y = np.array(y)
if X.ndim != 2:
raise ValueError('Wrong shape')
if manual_logtransform:
configspace = openmlpimp.utils.scale_configspace_to_log(configspace)
cutoffs = (-np.inf, np.inf)
if use_percentiles:
p75 = np.percentile(y, 75.0)
p100 = np.percentile(y, 100.0)
cutoffs = (p75, p100)
# start the evaluator
evaluator = fanova_pyrfr(X=X, Y=y, config_space=configspace, config_on_hypercube=False, cutoffs=cutoffs, n_trees=n_trees)
# obtain the results
params = configspace.get_hyperparameters()
result = {}
for idx, param in enumerate(params):
importance = evaluator.quantify_importance([idx])[(idx,)]['total importance']
result[param.name] = importance
# store main results to disk
filename = 'pimp_values_fanova.json'
with open(os.path.join(save_folder, filename), 'w') as out_file:
json.dump(result, out_file, sort_keys=True, indent=4, separators=(',', ': '))
print('Saved individuals to %s' %os.path.join(save_folder, filename))
# call plotting fn
yrange = (0, 1)
if use_percentiles:
yrange = (p75, p100)
if draw_plots:
FanovaBackend._plot_result(evaluator, configspace, save_folder + '/fanova', yrange)
if interaction_effect:
result_interaction = {}
for idx, param in enumerate(params):
for idx2, param2 in enumerate(params):
if param.name >= param2.name: # string comparison cause stable
continue
print('interaction effects between', param.name, param2.name)
interaction = evaluator.quantify_importance([idx, idx2])[(idx,idx2)]['total importance']
interaction -= result[param.name]
interaction -= result[param2.name]
combined_name = param.name + '__' + param2.name
if interaction < 0.0:
raise ValueError('interaction score too low. Params: %s score %d' %(combined_name, interaction))
result_interaction[combined_name] = interaction
for idx, param in enumerate(params):
for idx2, param2 in enumerate(params):
if param.name >= param2.name: # string comparison cause stable
continue
for idx3, param3 in enumerate(params):
if param2.name >= param3.name: # string comparison cause stable
continue
print('interaction effects between', param.name, param2.name, param3.name)
interaction = evaluator.quantify_importance([idx, idx2, idx3])[(idx, idx2, idx3)]['total importance']
interaction -= result[param.name]
interaction -= result[param2.name]
interaction -= result[param3.name]
combined_name = param.name + '__' + param2.name + '__' + param3.name
interaction -= result_interaction[param.name + '__' + param2.name]
interaction -= result_interaction[param2.name + '__' + param3.name]
interaction -= result_interaction[param.name + '__' + param3.name]
if interaction < 0.0:
raise ValueError('interaction score too low. Params: %s score %d' % (combined_name, interaction))
result_interaction[combined_name] = interaction
# store interaction effects to disk
if sum(result_interaction.values()) + sum(result.values()) > 1:
raise ValueError('Sum of results too high')
filename = 'pimp_values_fanova_interaction.json'
with open(os.path.join(save_folder, filename), 'w') as out_file:
json.dump(result_interaction, out_file, sort_keys=True, indent=4, separators=(',', ': '))
print('Saved interactions to %s' %os.path.join(save_folder, filename))
if draw_plots:
vis = Visualizer(evaluator, configspace, save_folder + '/fanova', y_label='Predictive Accuracy')
vis.create_most_important_pairwise_marginal_plots()
return save_folder + "/" + filename
def test_read_new_configuration_space_complex_conditionals(self):
classi = OrdinalHyperparameter(
"classi",
[
"random_forest", "extra_trees", "k_nearest_neighbors",
"something"
],
)
knn_weights = CategoricalHyperparameter("knn_weights",
["uniform", "distance"])
weather = OrdinalHyperparameter(
"weather", ["sunny", "rainy", "cloudy", "snowing"])
temperature = CategoricalHyperparameter("temperature", ["high", "low"])
rain = CategoricalHyperparameter("rain", ["yes", "no"])
gloves = OrdinalHyperparameter("gloves",
["none", "yarn", "leather", "gortex"])
heur1 = CategoricalHyperparameter("heur1", ["off", "on"])
heur2 = CategoricalHyperparameter("heur2", ["off", "on"])
heur_order = CategoricalHyperparameter("heur_order",
["heur1then2", "heur2then1"])
gloves_condition = OrConjunction(
EqualsCondition(gloves, rain, "yes"),
EqualsCondition(gloves, temperature, "low"))
heur_condition = AndConjunction(
EqualsCondition(heur_order, heur1, "on"),
EqualsCondition(heur_order, heur2, "on"))
and_conjunction = AndConjunction(
NotEqualsCondition(knn_weights, classi, "extra_trees"),
EqualsCondition(knn_weights, classi, "random_forest"))
Cl_condition = OrConjunction(
EqualsCondition(knn_weights, classi, "k_nearest_neighbors"),
and_conjunction, EqualsCondition(knn_weights, classi, "something"))
and1 = AndConjunction(EqualsCondition(temperature, weather, "rainy"),
EqualsCondition(temperature, weather, "cloudy"))
and2 = AndConjunction(
EqualsCondition(temperature, weather, "sunny"),
NotEqualsCondition(temperature, weather, "snowing"))
another_condition = OrConjunction(and1, and2)
complex_conditional_space = ConfigurationSpace()
complex_conditional_space.add_hyperparameter(classi)
complex_conditional_space.add_hyperparameter(knn_weights)
complex_conditional_space.add_hyperparameter(weather)
complex_conditional_space.add_hyperparameter(temperature)
complex_conditional_space.add_hyperparameter(rain)
complex_conditional_space.add_hyperparameter(gloves)
complex_conditional_space.add_hyperparameter(heur1)
complex_conditional_space.add_hyperparameter(heur2)
complex_conditional_space.add_hyperparameter(heur_order)
complex_conditional_space.add_condition(gloves_condition)
complex_conditional_space.add_condition(heur_condition)
complex_conditional_space.add_condition(Cl_condition)
complex_conditional_space.add_condition(another_condition)
complex_cs = list()
complex_cs.append(
"classi ordinal {random_forest,extra_trees,k_nearest_neighbors, something} "
"[random_forest]")
complex_cs.append(
"knn_weights categorical {uniform, distance} [uniform]")
complex_cs.append(
"weather ordinal {sunny, rainy, cloudy, snowing} [sunny]")
complex_cs.append("temperature categorical {high, low} [high]")
complex_cs.append("rain categorical { yes, no } [yes]")
complex_cs.append(
"gloves ordinal { none, yarn, leather, gortex } [none]")
complex_cs.append("heur1 categorical { off, on } [off]")
complex_cs.append("heur2 categorical { off, on } [off]")
complex_cs.append(
"heur_order categorical { heur1then2, heur2then1 } [heur1then2]")
complex_cs.append("gloves | rain == yes || temperature == low")
complex_cs.append("heur_order | heur1 == on && heur2 == on")
complex_cs.append(
"knn_weights | classi == k_nearest_neighbors || "
"classi != extra_trees && classi == random_forest || classi == something"
)
complex_cs.append(
"temperature | weather == rainy && weather == cloudy || "
"weather == sunny && weather != snowing")
cs_new = pcs_new.read(complex_cs)
self.assertEqual(cs_new, complex_conditional_space)
def handle_update_search_space(self, data):
"""change json format to ConfigSpace format dict<dict> -> configspace
Parameters
----------
data: JSON object
search space of this experiment
"""
search_space = data
cs = None
logger.debug(f'Received data: {data}')
if self.config_space:
logger.info(
f'Got a ConfigSpace file path, parsing the search space directly from {self.config_space}. '
'The NNI search space is ignored.')
with open(self.config_space, 'r') as fh:
cs = pcs_new.read(fh)
else:
cs = CS.ConfigurationSpace()
for var in search_space:
_type = str(search_space[var]["_type"])
if _type == 'choice':
cs.add_hyperparameter(
CSH.CategoricalHyperparameter(
var, choices=search_space[var]["_value"]))
elif _type == 'randint':
cs.add_hyperparameter(
CSH.UniformIntegerHyperparameter(
var,
lower=search_space[var]["_value"][0],
upper=search_space[var]["_value"][1] - 1))
elif _type == 'uniform':
cs.add_hyperparameter(
CSH.UniformFloatHyperparameter(
var,
lower=search_space[var]["_value"][0],
upper=search_space[var]["_value"][1]))
elif _type == 'quniform':
cs.add_hyperparameter(
CSH.UniformFloatHyperparameter(
var,
lower=search_space[var]["_value"][0],
upper=search_space[var]["_value"][1],
q=search_space[var]["_value"][2]))
elif _type == 'loguniform':
cs.add_hyperparameter(
CSH.UniformFloatHyperparameter(
var,
lower=search_space[var]["_value"][0],
upper=search_space[var]["_value"][1],
log=True))
elif _type == 'qloguniform':
cs.add_hyperparameter(
CSH.UniformFloatHyperparameter(
var,
lower=search_space[var]["_value"][0],
upper=search_space[var]["_value"][1],
q=search_space[var]["_value"][2],
log=True))
elif _type == 'normal':
cs.add_hyperparameter(
CSH.NormalFloatHyperparameter(
var,
mu=search_space[var]["_value"][1],
sigma=search_space[var]["_value"][2]))
elif _type == 'qnormal':
cs.add_hyperparameter(
CSH.NormalFloatHyperparameter(
var,
mu=search_space[var]["_value"][1],
sigma=search_space[var]["_value"][2],
q=search_space[var]["_value"][3]))
elif _type == 'lognormal':
cs.add_hyperparameter(
CSH.NormalFloatHyperparameter(
var,
mu=search_space[var]["_value"][1],
sigma=search_space[var]["_value"][2],
log=True))
elif _type == 'qlognormal':
cs.add_hyperparameter(
CSH.NormalFloatHyperparameter(
var,
mu=search_space[var]["_value"][1],
sigma=search_space[var]["_value"][2],
q=search_space[var]["_value"][3],
log=True))
else:
raise ValueError(
'unrecognized type in search_space, type is {}'.format(
_type))
self.search_space = cs
def load_configspace(self, folder):
"""Will try to load the configspace. cs_options will be a list containing all possible
combinations of interpretation for Categoricals. If this issue will be fixed, we can drop this procedure.
Parameters
----------
folder: str
path to folder in which to look for configspace
Returns
-------
cs_options: list[ConfigurationSpace]
list with possible interpretations for config-space-file. Only contains multiple items if file-format is pcs.
"""
cs_options = []
cs_fn_json = os.path.join(folder, 'configspace.json')
cs_fn_pcs = os.path.join(folder, 'configspace.pcs')
if os.path.exists(cs_fn_json):
with open(cs_fn_json, 'r') as fh:
cs_options = [pcs_json.read(fh.read())]
self.logger.debug(
"Detected and loaded \"%s\". No alternative interpretations necessary",
cs_fn_json)
elif os.path.exists(cs_fn_pcs):
with open(cs_fn_pcs, 'r') as fh:
cs = pcs_new.read(fh.readlines())
# Create alternative interpretations
categoricals = [
hp for hp in cs.get_hyperparameters()
if isinstance(hp, CategoricalHyperparameter)
]
non_categoricals = [
hp for hp in cs.get_hyperparameters()
if not isinstance(hp, CategoricalHyperparameter)
]
def _get_interpretations(choices):
""" Generate different interpretations for critical categorical hyperparameters that are not seamlessly
supported by pcs-format."""
result = []
if set(choices) == {"True", "False"}:
result.append([True, False])
if all([c.isdigit() for c in choices]):
result.append([int(c) for c in choices])
result.append(choices)
return result
choices_per_cat = [
_get_interpretations(hp.choices) for hp in categoricals
]
combinations = itertools.product(*choices_per_cat)
self.logger.debug(combinations)
for combi in combinations:
bcs = ConfigurationSpace()
for hp in non_categoricals:
bcs.add_hyperparameter(hp)
for name, choices in zip([hp.name for hp in categoricals],
combi):
bcs.add_hyperparameter(
CategoricalHyperparameter(name, choices))
bcs.add_conditions(cs.get_conditions())
cs_options.append(bcs)
self.logger.debug("Sampled %d interpretations of \"%s\"",
len(cs_options), cs_fn_pcs)
else:
raise ValueError("Missing pcs-file at '%s.[pcs|json]'!" %
os.path.join(folder, 'configspace'))
return cs_options
def get_configspace():
with open('searchspace.pcs', 'r') as fh:
cs = pcs_new.read(fh)
return cs
# TODO: make the default!
ignore_parameters = fixed_parameters_to_ignore_parameters(
args.fixed_parameters)
runhistory_path, configspace_path = openmlpimp.utils.cache_runhistory_configspace(
task_cache_folder,
args.flow_id,
task_id,
model_type=args.model_type,
required_setups=args.required_setups,
reverse=False,
fixed_parameters=args.fixed_parameters,
ignore_parameters=ignore_parameters)
if total_ranks is None:
with open(configspace_path) as configspace_file:
configspace = read(configspace_file)
total_ranks = {
param.name: 0
for param in configspace.get_hyperparameters()
}
if args.modus == 'fanova':
print('Running FANOVA backend on task %d' % task_id)
results_file = FanovaBackend.execute(
task_save_folder,
runhistory_path,
configspace_path,
use_percentiles=args.use_quantiles,
interaction_effect=args.interaction_effect,
n_trees=args.n_trees,
run_limit=args.limit,
