def run_test(self):
try:
with open(configuration_space_path) as fh:
cs = pcs_parser.read(fh)
except:
with open(configuration_space_path) as fh:
cs = pcs_new_parser.read(fh)
default = cs.get_default_configuration()
cs._check_configuration_rigorous(default)
for i in range(10):
neighborhood = ConfigSpaceNNI.util.get_one_exchange_neighbourhood(
default, seed=i)
for shuffle, n in enumerate(neighborhood):
n.is_valid_configuration()
cs._check_configuration_rigorous(n)
if shuffle == 10:
break
# Sample a little bit
for i in range(10):
cs.seed(i)
configurations = cs.sample_configuration(size=5)
for j, c in enumerate(configurations):
c.is_valid_configuration()
cs._check_configuration_rigorous(c)
neighborhood = ConfigSpaceNNI.util.get_one_exchange_neighbourhood(
c, seed=i)
for shuffle, n in enumerate(neighborhood):
n.is_valid_configuration()
cs._check_configuration_rigorous(n)
if shuffle == 20:
break
def test_read_configuration_space_conditional_with_two_parents(self):
config_space = list()
config_space.append("@1:0:restarts {F,L,D,x,+,no}[x]")
config_space.append("@1:S:Luby:aryrestarts {1,2}[1]")
config_space.append("@1:2:Luby:restarts [1,65535][1000]il")
config_space.append("@1:2:Luby:restarts | @1:0:restarts in {L}")
config_space.append(
"@1:2:Luby:restarts | @1:S:Luby:aryrestarts in {2}")
cs = pcs.read(config_space)
self.assertEqual(len(cs.get_conditions()), 1)
self.assertIsInstance(cs.get_conditions()[0], AndConjunction)
def test_random_neigbor_conditional(self):
mini_autosklearn_config_space_path = os.path.join(
os.path.dirname(__file__), 'test_searchspaces',
'mini_autosklearn_original.pcs')
with open(mini_autosklearn_config_space_path) as fh:
cs = read(fh)
cs.seed(1)
configuration = cs.get_default_configuration()
for i in range(100):
new_config = get_random_neighbor(configuration, i)
self.assertNotEqual(configuration, new_config)
def test_spear(self):
'''
simply getting some random configuration from spear pcs
'''
file_path = os.path.join(os.path.dirname(__file__), '..',
'test_files', 'spear-params.pcs')
with open(file_path) as fp:
pcs_str = fp.readlines()
cs = pcs.read(pcs_str)
for i in range(100):
config = cs.sample_configuration()
print(config.get_dictionary())
def test_read_write(self):
# Some smoke tests whether reading, writing, reading alters makes the
# configspace incomparable
this_file = os.path.abspath(__file__)
this_directory = os.path.dirname(this_file)
configuration_space_path = os.path.join(this_directory, "..",
"test_searchspaces")
configuration_space_path = os.path.abspath(configuration_space_path)
configuration_space_path = os.path.join(configuration_space_path,
"spear-params-mixed.pcs")
with open(configuration_space_path) as fh:
cs = pcs.read(fh)
tf = tempfile.NamedTemporaryFile()
name = tf.name
tf.close()
with open(name, 'w') as fh:
pcs_string = pcs.write(cs)
fh.write(pcs_string)
with open(name, 'r') as fh:
pcs_new = pcs.read(fh)
self.assertEqual(pcs_new, cs, msg=(pcs_new, cs))
def test_impute_inactive_values(self):
mini_autosklearn_config_space_path = os.path.join(
os.path.dirname(__file__), 'test_searchspaces',
'mini_autosklearn_original.pcs')
with open(mini_autosklearn_config_space_path) as fh:
cs = read(fh)
cs.seed(1)
configuration = cs.sample_configuration()
new_configuration = impute_inactive_values(configuration)
self.assertNotEqual(id(configuration), id(new_configuration))
self.assertEqual(len(new_configuration._values), 11)
for key in new_configuration:
self.assertIsNotNone(new_configuration[key])
self.assertEqual(new_configuration['random_forest:max_features'], 10)
def run_test(configuration_space_path):
if not '2017_11' in configuration_space_path:
return
with open(configuration_space_path) as fh:
cs = pcs_parser.read(fh)
print('###')
print(configuration_space_path, flush=True)
sampling_time = []
neighborhood_time = []
validation_times = []
# Sample a little bit
for i in range(10):
cs.seed(i)
start_time = time.time()
configurations = cs.sample_configuration(size=n_configs)
end_time = time.time()
sampling_time.append(end_time - start_time)
for j, c in enumerate(configurations):
#c.is_valid_configuration()
if i == 0:
neighborhood = ConfigSpaceNNI.util.get_one_exchange_neighbourhood(
c, seed=i * j)
start_time = time.time()
validation_time = []
for shuffle, n in enumerate(neighborhood):
v_start_time = time.time()
n.is_valid_configuration()
v_end_time = time.time()
validation_time.append(v_end_time - v_start_time)
if shuffle == 10:
break
end_time = time.time()
neighborhood_time.append(end_time - start_time -
np.sum(validation_time))
validation_times.extend(validation_time)
print('Average time sampling %d configurations' % n_configs,
np.mean(sampling_time))
print('Average time retrieving a nearest neighbor',
np.mean(neighborhood_time))
print('Average time checking one configuration', np.mean(validation_times))
def test_read_configuration_space_easy(self):
expected = StringIO()
expected.write('# This is a \n')
expected.write(
' # This is a comment with a leading whitespace ### ffds \n')
expected.write('\n')
expected.write('float_a [-1.23, 6.45] [2.61] # bla\n')
expected.write('e_float_a [.5E-2, 4.5e+06] [2250000.0025]\n')
expected.write('int_a [-1, 6] [2]i\n')
expected.write('log_a [4e-1, 6.45] [1.6062378404]l\n')
expected.write('int_log_a [1, 6] [2]il\n')
expected.write('cat_a {a,"b",c,d} [a]\n')
expected.write('@.:;/\?!$%&_-<>*+1234567890 {"const"} ["const"]\n')
expected.seek(0)
cs = pcs.read(expected)
self.assertEqual(cs, easy_space)
def test_read_new_configuration_space_conditional(self):
# More complex search space as string array
complex_cs = list()
complex_cs.append("preprocessing categorical {None, pca} [None]")
complex_cs.append("classifier categorical {svm, nn} [svm]")
complex_cs.append("kernel categorical {rbf, poly, sigmoid} [rbf]")
complex_cs.append("C real [0.03125, 32768] [32]log")
complex_cs.append("neurons integer [16, 1024] [520] # Should be Q16")
complex_cs.append("lr real [0.0001, 1.0] [0.50005]")
complex_cs.append("degree integer [1, 5] [3]")
complex_cs.append("gamma real [0.000030518, 8] [0.0156251079996]log")
complex_cs.append("C | classifier in {svm}")
complex_cs.append("kernel | classifier in {svm}")
complex_cs.append("lr | classifier in {nn}")
complex_cs.append("neurons | classifier in {nn}")
complex_cs.append("degree | kernel in {poly, sigmoid}")
complex_cs.append("gamma | kernel in {rbf}")
cs_new = pcs_new.read(complex_cs)
self.assertEqual(cs_new, conditional_space)
# same in older version
complex_cs_old = list()
complex_cs_old.append("preprocessing {None, pca} [None]")
complex_cs_old.append("classifier {svm, nn} [svm]")
complex_cs_old.append("kernel {rbf, poly, sigmoid} [rbf]")
complex_cs_old.append("C [0.03125, 32768] [32]l")
complex_cs_old.append("neurons [16, 1024] [520]i # Should be Q16")
complex_cs_old.append("lr [0.0001, 1.0] [0.50005]")
complex_cs_old.append("degree [1, 5] [3]i")
complex_cs_old.append("gamma [0.000030518, 8] [0.0156251079996]l")
complex_cs_old.append("C | classifier in {svm}")
complex_cs_old.append("kernel | classifier in {svm}")
complex_cs_old.append("lr | classifier in {nn}")
complex_cs_old.append("neurons | classifier in {nn}")
complex_cs_old.append("degree | kernel in {poly, sigmoid}")
complex_cs_old.append("gamma | kernel in {rbf}")
cs_old = pcs.read(complex_cs_old)
self.assertEqual(cs_old, cs_new)
def test_read_configuration_space_conditional(self):
# More complex search space as string array
complex_cs = list()
complex_cs.append("preprocessing {None, pca} [None]")
complex_cs.append("classifier {svm, nn} [svm]")
complex_cs.append("kernel {rbf, poly, sigmoid} [rbf]")
complex_cs.append("C [0.03125, 32768] [32]l")
complex_cs.append("neurons [16, 1024] [520]i # Should be Q16")
complex_cs.append("lr [0.0001, 1.0] [0.50005]")
complex_cs.append("degree [1, 5] [3]i")
complex_cs.append("gamma [0.000030518, 8] [0.0156251079996]l")
complex_cs.append("C | classifier in {svm}")
complex_cs.append("kernel | classifier in {svm}")
complex_cs.append("lr | classifier in {nn}")
complex_cs.append("neurons | classifier in {nn}")
complex_cs.append("degree | kernel in {poly, sigmoid}")
complex_cs.append("gamma | kernel in {rbf}")
cs = pcs.read(complex_cs)
self.assertEqual(cs, conditional_space)