def test_read_configuration_space_conditional_with_two_parents(self):
config_space = list()
config_space.append("@1:0:restarts {F,L,D,x,+,no}[x]")
config_space.append("@1:S:Luby:aryrestarts {1,2}[1]")
config_space.append("@1:2:Luby:restarts [1,65535][1000]il")
config_space.append("@1:2:Luby:restarts | @1:0:restarts in {L}")
config_space.append("@1:2:Luby:restarts | @1:S:Luby:aryrestarts in {2}")
cs = pcs.read(config_space)
self.assertEqual(len(cs.get_conditions()), 1)
self.assertIsInstance(cs.get_conditions()[0], AndConjunction)
def test_random_neigbor_conditional(self):
mini_autosklearn_config_space_path = os.path.join(
os.path.dirname(__file__), 'test_searchspaces',
'mini_autosklearn_original.pcs')
with open(mini_autosklearn_config_space_path) as fh:
cs = read(fh)
cs.seed(1)
configuration = cs.get_default_configuration()
for i in range(100):
new_config = get_random_neighbor(configuration, i)
self.assertNotEqual(configuration, new_config)
def test_read_write(self):
# Some smoke tests whether reading, writing, reading alters makes the
# configspace incomparable
this_file = os.path.abspath(__file__)
this_directory = os.path.dirname(this_file)
configuration_space_path = os.path.join(this_directory,
"..", "test_searchspaces")
configuration_space_path = os.path.abspath(configuration_space_path)
configuration_space_path = os.path.join(configuration_space_path,
"spear-params-mixed.pcs")
with open(configuration_space_path) as fh:
cs = pcs.read(fh)
tf = tempfile.NamedTemporaryFile()
name = tf.name
tf.close()
with open(name, 'w') as fh:
pcs_string = pcs.write(cs)
fh.write(pcs_string)
with open(name, 'r') as fh:
pcs_new = pcs.read(fh)
assert (pcs_new, cs)
def test_random_neigborhood_conditional(self):
mini_autosklearn_config_space_path = os.path.join(
os.path.dirname(__file__), 'test_searchspaces',
'mini_autosklearn_original.pcs')
with open(mini_autosklearn_config_space_path) as fh:
cs = read(fh)
cs.seed(1)
configuration = cs.get_default_configuration()
for i in range(100):
neighborhood = get_one_exchange_neighbourhood(configuration, i)
for new_config in neighborhood:
self.assertNotEqual(configuration, new_config)
def test_read_write(self):
# Some smoke tests whether reading, writing, reading alters makes the
# configspace incomparable
this_file = os.path.abspath(__file__)
this_directory = os.path.dirname(this_file)
configuration_space_path = os.path.join(this_directory, "..",
"test_searchspaces")
configuration_space_path = os.path.abspath(configuration_space_path)
configuration_space_path = os.path.join(configuration_space_path,
"spear-params-mixed.pcs")
with open(configuration_space_path) as fh:
cs = pcs.read(fh)
tf = tempfile.NamedTemporaryFile()
name = tf.name
tf.close()
with open(name, 'w') as fh:
pcs_string = pcs.write(cs)
fh.write(pcs_string)
with open(name, 'r') as fh:
pcs_new = pcs.read(fh)
assert (pcs_new, cs)
def test_spear(self):
'''
simply getting some random configuration from spear pcs
'''
file_path = os.path.join(os.path.dirname(__file__), '..', 'test_files',
'spear-params.pcs')
with open(file_path) as fp:
pcs_str = fp.readlines()
cs = pcs.read(pcs_str)
for i in range(100):
config = cs.sample_configuration()
print(config.get_dictionary())
def test_read_configuration_space_easy(self):
expected = StringIO()
expected.write('# This is a \n')
expected.write(' # This is a comment with a leading whitespace ### ffds \n')
expected.write('\n')
expected.write('float_a [-1.23, 6.45] [2.61] # bla\n')
expected.write('e_float_a [.5E-2, 4.5e+06] [2250000.0025]\n')
expected.write('int_a [-1, 6] [2]i\n')
expected.write('log_a [4e-1, 6.45] [1.6062378404]l\n')
expected.write('int_log_a [1, 6] [2]il\n')
expected.write('cat_a {a,"b",c,d} [a]\n')
expected.write('@.:;/\?!$%&_-<>*+1234567890 {"const"} ["const"]\n')
expected.seek(0)
cs = pcs.read(expected)
self.assertEqual(cs, easy_space)
def test_impute_inactive_values(self):
mini_autosklearn_config_space_path = os.path.join(
os.path.dirname(__file__), 'test_searchspaces',
'mini_autosklearn_original.pcs')
with open(mini_autosklearn_config_space_path) as fh:
cs = read(fh)
cs.seed(1)
configuration = cs.sample_configuration()
new_configuration = impute_inactive_values(configuration)
self.assertNotEqual(id(configuration), id(new_configuration))
self.assertEqual(len(new_configuration._values), 11)
for key in new_configuration:
self.assertIsNotNone(new_configuration[key])
self.assertEqual(new_configuration['random_forest:max_features'], 10)
def test_impute_inactive_values(self):
mini_autosklearn_config_space_path = os.path.join(
os.path.dirname(__file__), 'test_searchspaces',
'mini_autosklearn_original.pcs')
with open(mini_autosklearn_config_space_path) as fh:
cs = read(fh)
cs.seed(1)
configuration = cs.sample_configuration()
new_configuration = impute_inactive_values(configuration)
self.assertNotEqual(id(configuration), id(new_configuration))
self.assertEqual(len(new_configuration._values), 11)
for key in new_configuration:
self.assertIsNotNone(new_configuration[key])
self.assertEqual(new_configuration['random_forest:max_features'], 10)
def test_read_configuration_space_easy(self):
expected = six.StringIO()
expected.write('# This is a \n')
expected.write(
' # This is a comment with a leading whitespace ### ffds \n')
expected.write('\n')
expected.write('float_a [-1.23, 6.45] [2.61] # bla\n')
expected.write('e_float_a [.5E-2, 4.5e+06] [2250000.0025]\n')
expected.write('int_a [-1, 6] [2]i\n')
expected.write('log_a [4e-1, 6.45] [1.6062378404]l\n')
expected.write('int_log_a [1, 6] [2]il\n')
expected.write('cat_a {a,"b",c,d} [a]\n')
expected.write('@.:;/\?!$%&_-<>*+1234567890 {"const"} ["const"]\n')
expected.seek(0)
cs = pcs.read(expected)
self.assertEqual(cs, easy_space)
def run_test(configuration_space_path):
if not '2017' in configuration_space_path:
return
with open(configuration_space_path) as fh:
cs = pcs_parser.read(fh)
print('###')
print(configuration_space_path, flush=True)
sampling_time = []
neighborhood_time = []
validation_times = []
# Sample a little bit
for i in range(10):
cs.seed(i)
start_time = time.time()
configurations = cs.sample_configuration(size=100)
end_time = time.time()
sampling_time.append(end_time - start_time)
for j, c in enumerate(configurations):
c.is_valid_configuration()
if i == 0:
neighborhood = ConfigSpace.util.get_one_exchange_neighbourhood(
c, seed=i*j)
start_time = time.time()
validation_time = []
for shuffle, n in enumerate(neighborhood):
v_start_time = time.time()
n.is_valid_configuration()
v_end_time = time.time()
validation_time.append(v_end_time - v_start_time)
if shuffle == 10:
break
end_time = time.time()
neighborhood_time.append(end_time - start_time - np.sum(validation_time))
validation_times.extend(validation_time)
print('Average time sampling 100 configurations', np.mean(sampling_time))
print('Average time retrieving a nearest neighbor', np.mean(neighborhood_time))
print('Average time checking one configuration', np.mean(validation_times))
def run_test(self):
if 'autoweka' in configuration_space_path:
return
with open(configuration_space_path) as fh:
cs = pcs_parser.read(fh)
# Sample a little bit
for i in range(10):
print(i)
cs.seed(i)
configurations = cs.sample_configuration(size=10)
for j, c in enumerate(configurations):
print(j)
c.is_valid_configuration()
neighborhood = ConfigSpace.util.get_one_exchange_neighbourhood(
c, seed=i)
for n in neighborhood:
n.is_valid_configuration()
def run_test(self):
with open(configuration_space_path) as fh:
cs = pcs_parser.read(fh)
# Sample a little bit
for i in range(10):
print(i)
cs.seed(i)
configurations = cs.sample_configuration(size=5)
for j, c in enumerate(configurations):
print(j)
c.is_valid_configuration()
neighborhood = ConfigSpace.util.get_one_exchange_neighbourhood(
c, seed=i)
for shuffle, n in enumerate(neighborhood):
n.is_valid_configuration()
if shuffle == 10:
break
def test_read_new_configuration_space_conditional(self):
# More complex search space as string array
complex_cs = list()
complex_cs.append("preprocessing categorical {None, pca} [None]")
complex_cs.append("classifier categorical {svm, nn} [svm]")
complex_cs.append("kernel categorical {rbf, poly, sigmoid} [rbf]")
complex_cs.append("C real [0.03125, 32768] [32]log")
complex_cs.append("neurons integer [16, 1024] [520] # Should be Q16")
complex_cs.append("lr real [0.0001, 1.0] [0.50005]")
complex_cs.append("degree integer [1, 5] [3]")
complex_cs.append("gamma real [0.000030518, 8] [0.0156251079996]log")
complex_cs.append("C | classifier in {svm}")
complex_cs.append("kernel | classifier in {svm}")
complex_cs.append("lr | classifier in {nn}")
complex_cs.append("neurons | classifier in {nn}")
complex_cs.append("degree | kernel in {poly, sigmoid}")
complex_cs.append("gamma | kernel in {rbf}")
cs_new = pcs_new.read(complex_cs)
self.assertEqual(cs_new, conditional_space)
# same in older version
complex_cs_old = list()
complex_cs_old.append("preprocessing {None, pca} [None]")
complex_cs_old.append("classifier {svm, nn} [svm]")
complex_cs_old.append("kernel {rbf, poly, sigmoid} [rbf]")
complex_cs_old.append("C [0.03125, 32768] [32]l")
complex_cs_old.append("neurons [16, 1024] [520]i # Should be Q16")
complex_cs_old.append("lr [0.0001, 1.0] [0.50005]")
complex_cs_old.append("degree [1, 5] [3]i")
complex_cs_old.append("gamma [0.000030518, 8] [0.0156251079996]l")
complex_cs_old.append("C | classifier in {svm}")
complex_cs_old.append("kernel | classifier in {svm}")
complex_cs_old.append("lr | classifier in {nn}")
complex_cs_old.append("neurons | classifier in {nn}")
complex_cs_old.append("degree | kernel in {poly, sigmoid}")
complex_cs_old.append("gamma | kernel in {rbf}")
cs_old = pcs.read(complex_cs_old)
self.assertEqual(cs_old, cs_new)
def test_read_new_configuration_space_conditional(self):
# More complex search space as string array
complex_cs = list()
complex_cs.append("preprocessing categorical {None, pca} [None]")
complex_cs.append("classifier categorical {svm, nn} [svm]")
complex_cs.append("kernel categorical {rbf, poly, sigmoid} [rbf]")
complex_cs.append("C real [0.03125, 32768] [32]log")
complex_cs.append("neurons integer [16, 1024] [520] # Should be Q16")
complex_cs.append("lr real [0.0001, 1.0] [0.50005]")
complex_cs.append("degree integer [1, 5] [3]")
complex_cs.append("gamma real [0.000030518, 8] [0.0156251079996]log")
complex_cs.append("C | classifier in {svm}")
complex_cs.append("kernel | classifier in {svm}")
complex_cs.append("lr | classifier in {nn}")
complex_cs.append("neurons | classifier in {nn}")
complex_cs.append("degree | kernel in {poly, sigmoid}")
complex_cs.append("gamma | kernel in {rbf}")
cs_new = pcs_new.read(complex_cs)
self.assertEqual(cs_new, conditional_space)
# same in older version
complex_cs_old = list()
complex_cs_old.append("preprocessing {None, pca} [None]")
complex_cs_old.append("classifier {svm, nn} [svm]")
complex_cs_old.append("kernel {rbf, poly, sigmoid} [rbf]")
complex_cs_old.append("C [0.03125, 32768] [32]l")
complex_cs_old.append("neurons [16, 1024] [520]i # Should be Q16")
complex_cs_old.append("lr [0.0001, 1.0] [0.50005]")
complex_cs_old.append("degree [1, 5] [3]i")
complex_cs_old.append("gamma [0.000030518, 8] [0.0156251079996]l")
complex_cs_old.append("C | classifier in {svm}")
complex_cs_old.append("kernel | classifier in {svm}")
complex_cs_old.append("lr | classifier in {nn}")
complex_cs_old.append("neurons | classifier in {nn}")
complex_cs_old.append("degree | kernel in {poly, sigmoid}")
complex_cs_old.append("gamma | kernel in {rbf}")
cs_old = pcs.read(complex_cs_old)
self.assertEqual(cs_old, cs_new)
def test_read_configuration_space_conditional(self):
# More complex search space as string array
complex_cs = list()
complex_cs.append("preprocessing {None, pca} [None]")
complex_cs.append("classifier {svm, nn} [svm]")
complex_cs.append("kernel {rbf, poly, sigmoid} [rbf]")
complex_cs.append("C [0.03125, 32768] [32]l")
complex_cs.append("neurons [16, 1024] [520]i # Should be Q16")
complex_cs.append("lr [0.0001, 1.0] [0.50005]")
complex_cs.append("degree [1, 5] [3]i")
complex_cs.append("gamma [0.000030518, 8] [0.0156251079996]l")
complex_cs.append("C | classifier in {svm}")
complex_cs.append("kernel | classifier in {svm}")
complex_cs.append("lr | classifier in {nn}")
complex_cs.append("neurons | classifier in {nn}")
complex_cs.append("degree | kernel in {poly, sigmoid}")
complex_cs.append("gamma | kernel in {rbf}")
cs = pcs.read(complex_cs)
self.assertEqual(cs, conditional_space)
def test_read_configuration_space_conditional(self):
# More complex search space as string array
complex_cs = list()
complex_cs.append("preprocessing {None, pca} [None]")
complex_cs.append("classifier {svm, nn} [svm]")
complex_cs.append("kernel {rbf, poly, sigmoid} [rbf]")
complex_cs.append("C [0.03125, 32768] [32]l")
complex_cs.append("neurons [16, 1024] [520]i # Should be Q16")
complex_cs.append("lr [0.0001, 1.0] [0.50005]")
complex_cs.append("degree [1, 5] [3]i")
complex_cs.append("gamma [0.000030518, 8] [0.0156251079996]l")
complex_cs.append("C | classifier in {svm}")
complex_cs.append("kernel | classifier in {svm}")
complex_cs.append("lr | classifier in {nn}")
complex_cs.append("neurons | classifier in {nn}")
complex_cs.append("degree | kernel in {poly, sigmoid}")
complex_cs.append("gamma | kernel in {rbf}")
cs = pcs.read(complex_cs)
self.assertEqual(cs, conditional_space)
def __init__(self, scen, ID, repetition=0, updates=""):
""" This function creates a configuration with default parameters, which will
(at the end of the function) be overwritten with the values in the updates-
dictionary. """
if isinstance(updates, str):
updates = self.dict_from_file(scen, ID)
elif not isinstance(updates, dict):
raise ValueError("updates to Config must be of type str (for"
"filepath) or dict.")
with open("dlas/dlas.pcs", 'r') as f:
configspace = pcs.read(f.readlines())
self.default_config = configspace.get_default_configuration()
config_dict = self.default_config.get_dictionary()
config_dict.update(updates)
self.config = Configuration(configspace, config_dict)
self.scen = scen
self.ID = ID
self.rep = repetition
self.use_validation = True
self.result_path = "results/{}/{}/{}/".format(self.scen, self.ID,
self.rep)
def run_test(self):
try:
with open(configuration_space_path) as fh:
cs = pcs_parser.read(fh)
except:
with open(configuration_space_path) as fh:
cs = pcs_new_parser.read(fh)
# Sample a little bit
for i in range(10):
cs.seed(i)
configurations = cs.sample_configuration(size=5)
for j, c in enumerate(configurations):
c.is_valid_configuration()
cs._check_configuration_rigorous(c)
neighborhood = ConfigSpace.util.get_one_exchange_neighbourhood(
c, seed=i)
for shuffle, n in enumerate(neighborhood):
n.is_valid_configuration()
cs._check_configuration_rigorous(n)
if shuffle == 10:
break
parser = argparse.ArgumentParser()
parser.add_argument('configuration_space', type=str)
parser.add_argument('output_directory', type=str)
parser.add_argument('-n', default=10, type=int)
args = parser.parse_args()
config_space_path = args.configuration_space
output_directory = args.output_directory
n = args.n
with open(config_space_path) as fh:
config_space_string = fh.read().split('\n')
config_space = pcs.read(config_space_string)
try:
os.makedirs(output_directory)
except:
pass
if not os.path.exists(output_directory):
print('Could not create output directory.')
exit(1)
configurations = config_space.sample_configuration(n)
for i, configuration in enumerate(configurations):
values = configuration.get_dictionary()
new_values = {}
