def test_deterministic(self):
"""
Testing deterministic behaviour.
"""
testargs = [
"scripts/smac", "--scenario", self.scenario_file,
"--verbose_level", "DEBUG", "--seed", "1", "--output_dir",
self.output_dir_1
]
with mock.patch.object(sys, 'argv', testargs):
SMACCLI().main_cli()
testargs = [
"scripts/smac", "--scenario", self.scenario_file,
"--verbose_level", "DEBUG", "--seed", "1", "--output_dir",
self.output_dir_2
]
with mock.patch.object(sys, 'argv', testargs):
SMACCLI().main_cli()
testargs = [
"scripts/smac", "--scenario", self.scenario_file,
"--verbose_level", "DEBUG", "--seed", "2", "--output_dir",
self.output_dir_3
]
with mock.patch.object(sys, 'argv', testargs):
SMACCLI().main_cli()
# compare trajectories in output_dir_{1,2,3}
h1 = json.load(open(self.output_dir_1 + '/run_1/runhistory.json'))
h2 = json.load(open(self.output_dir_2 + '/run_1/runhistory.json'))
h3 = json.load(open(self.output_dir_3 + '/run_2/runhistory.json'))
self.assertEqual(h1, h2)
self.assertNotEqual(h1, h3)
def setUp(self):
# TODO after merging PR #264 (flat folder hierarchy), this will fail.
# simply adjust path and remove this note
self.output_one = "test/test_files/test_restore_state_run1" # From scenario_one.txt
self.output_two = "test/test_files/test_restored_state_run1" # From scenario_two.txt
self.smaccli = SMACCLI()
self.scenario_one = "test/test_files/restore_scenario_one.txt"
self.scenario_two = "test/test_files/restore_scenario_two.txt"
class TestSMACCLI(unittest.TestCase):
def setUp(self):
# TODO after merging PR #264 (flat folder hierarchy), this will fail.
# simply adjust path and remove this note
self.output_one = "test/test_files/test_restore_state_run1" # From scenario_one.txt
self.output_two = "test/test_files/test_restored_state_run1" # From scenario_two.txt
self.smaccli = SMACCLI()
self.scenario_one = "test/test_files/restore_scenario_one.txt"
self.scenario_two = "test/test_files/restore_scenario_two.txt"
def test_run_and_restore(self):
"""
Testing basic restore functionality.
"""
shutil.rmtree(self.output_one, ignore_errors=True)
shutil.rmtree(self.output_two, ignore_errors=True)
# Run for 5 algo-calls
testargs = ["python", "scripts/smac", "--scenario_file",
self.scenario_one, "--verbose", "DEBUG"]
with mock.patch.object(sys, 'argv', testargs):
self.smaccli.main_cli()
# Increase limit and run for 10 (so 5 more) by using restore_state
testargs = ["python", "scripts/smac", "--restore_state",
self.output_one, "--scenario_file",
self.scenario_two, "--verbose", "DEBUG"]
with mock.patch.object(sys, 'argv', testargs):
self.smaccli.main_cli()
def test_missing_dir(self):
"""
Testing error if dir is missing.
"""
testargs = ["python", "scripts/smac", "--restore_state",
"nonsense_test_dir", "--scenario_file",
self.scenario_two, "--verbose", "DEBUG"]
with mock.patch.object(sys, 'argv', testargs):
self.assertRaises(FileNotFoundError, self.smaccli.main_cli)
def test_illegal_input(self):
"""
Testing illegal input in smbo
"""
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter('test', 1, 10, 5))
scen = Scenario({'run_obj': 'quality', 'cs': cs})
stats = Stats(scen)
# Recorded runs but no incumbent.
stats.ta_runs = 10
smac = SMAC(scen, stats=stats, rng=np.random.RandomState(42))
self.assertRaises(ValueError, smac.optimize)
# Incumbent but no recoreded runs.
incumbent = cs.get_default_configuration()
smac = SMAC(scen, restore_incumbent=incumbent,
rng=np.random.RandomState(42))
self.assertRaises(ValueError, smac.optimize)
def setUp(self):
base_directory = os.path.split(__file__)[0]
base_directory = os.path.abspath(
os.path.join(base_directory, '..', '..'))
self.current_dir = os.getcwd()
os.chdir(base_directory)
self.output_one = "test/test_files/test_restore_state/run_1" # From scenario_one.txt
self.output_two = "test/test_files/test_restored_state/run_1" # From scenario_two.txt
self.smaccli = SMACCLI()
self.scenario_one = "test/test_files/restore_scenario_one.txt"
self.scenario_two = "test/test_files/restore_scenario_two.txt"
def test_deterministic(self, patch):
"""
Testing deterministic behaviour.
"""
# Make SMAC a bit faster
patch.side_effect = lambda configuration, seed: get_one_exchange_neighbourhood(
configuration=configuration,
stdev=0.05,
num_neighbors=2,
seed=seed,
)
testargs = [
"scripts/smac", "--scenario", self.scenario_file,
"--verbose_level", "DEBUG", "--seed", "1",
"--random_configuration_chooser",
"test/test_cli/random_configuration_chooser_impl.py",
"--output_dir", self.output_dir_1
]
SMACCLI().main_cli(testargs[1:])
testargs = [
"scripts/smac", "--scenario", self.scenario_file,
"--verbose_level", "DEBUG", "--seed", "1",
"--random_configuration_chooser",
"test/test_cli/random_configuration_chooser_impl.py",
"--output_dir", self.output_dir_2
]
SMACCLI().main_cli(testargs[1:])
testargs = [
"scripts/smac", "--scenario", self.scenario_file,
"--verbose_level", "DEBUG", "--seed", "2",
"--random_configuration_chooser",
"test/test_cli/random_configuration_chooser_impl.py",
"--output_dir", self.output_dir_3
]
SMACCLI().main_cli(testargs[1:])
# compare trajectories in output_dir_{1,2,3}
h1 = json.load(open(self.output_dir_1 + '/run_1/runhistory.json'))
h2 = json.load(open(self.output_dir_2 + '/run_1/runhistory.json'))
h3 = json.load(open(self.output_dir_3 + '/run_2/runhistory.json'))
self.assertEqual(self.ignore_timestamps(h1),
self.ignore_timestamps(h2))
# As h1 is changed inplace in the line above we need to reload it
h1 = json.load(open(self.output_dir_1 + '/run_1/runhistory.json'))
self.assertNotEqual(self.ignore_timestamps(h1),
self.ignore_timestamps(h3))
class TestSMACCLI(unittest.TestCase):
def setUp(self):
base_directory = os.path.split(__file__)[0]
base_directory = os.path.abspath(
os.path.join(base_directory, '..', '..'))
self.current_dir = os.getcwd()
os.chdir(base_directory)
self.output_one = "test/test_files/test_restore_state/run_1" # From scenario_one.txt
self.output_two = "test/test_files/test_restored_state/run_1" # From scenario_two.txt
self.smaccli = SMACCLI()
self.scenario_one = "test/test_files/restore_scenario_one.txt"
self.scenario_two = "test/test_files/restore_scenario_two.txt"
def tearDown(self):
shutil.rmtree(self.output_one, ignore_errors=True)
shutil.rmtree(self.output_two, ignore_errors=True)
os.chdir(self.current_dir)
def test_run_and_restore(self):
"""
Testing basic restore functionality.
"""
# Run for 5 algo-calls
testargs = [
"python", "scripts/smac", "--scenario_file", self.scenario_one,
"--verbose", "DEBUG"
]
with mock.patch.object(sys, 'argv', testargs):
self.smaccli.main_cli()
# Increase limit and run for 10 (so 5 more) by using restore_state
testargs = [
"python", "scripts/smac", "--restore_state", self.output_one,
"--scenario_file", self.scenario_two, "--verbose", "DEBUG"
]
with mock.patch.object(sys, 'argv', testargs):
self.smaccli.main_cli()
def test_missing_dir(self):
"""
Testing error if dir is missing.
"""
testargs = [
"python", "scripts/smac", "--restore_state", "nonsense_test_dir",
"--scenario_file", self.scenario_two, "--verbose", "DEBUG"
]
with mock.patch.object(sys, 'argv', testargs):
self.assertRaises(FileNotFoundError, self.smaccli.main_cli)
def test_illegal_input(self):
"""
Testing illegal input in smbo
"""
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter('test', 1, 10, 5))
scen = Scenario({'run_obj': 'quality', 'cs': cs})
stats = Stats(scen)
# Recorded runs but no incumbent.
stats.ta_runs = 10
smac = SMAC(scen, stats=stats, rng=np.random.RandomState(42))
self.assertRaises(ValueError, smac.optimize)
# Incumbent but no recoreded runs.
incumbent = cs.get_default_configuration()
smac = SMAC(scen,
restore_incumbent=incumbent,
rng=np.random.RandomState(42))
self.assertRaises(ValueError, smac.optimize)
def test_same_dir(self):
"""
Testing possible error using same dir for restore
"""
# Run for 5 algo-calls
testargs = [
"python", "scripts/smac", "--scenario_file", self.scenario_one,
"--verbose", "DEBUG"
]
with mock.patch.object(sys, 'argv', testargs):
self.smaccli.main_cli()
# Increase limit and run for 10 (so 5 more) by using restore_state
testargs = [
"python", "scripts/smac", "--restore_state", self.output_one,
"--scenario_file", self.scenario_one, "--verbose", "DEBUG"
]
with mock.patch.object(sys, 'argv', testargs):
self.smaccli.main_cli()
def test_modes(self):
"""
Test if different modes are accepted
"""
testargs = [
"scripts/smac", "--scenario", self.scenario_file,
"--verbose_level", "DEBUG", "--seed", "2",
"--random_configuration_chooser",
"test/test_cli/random_configuration_chooser_impl.py",
"--output_dir", self.output_dir_3, "--mode", 'SMAC4AC'
]
cli = SMACCLI()
with mock.patch("smac.smac_cli.SMAC4AC") as MSMAC:
MSMAC.return_value.optimize.return_value = True
cli.main_cli(testargs[1:])
MSMAC.assert_called_once_with(initial_configurations=None,
restore_incumbent=None,
run_id=2,
runhistory=None,
stats=None,
scenario=mock.ANY,
rng=mock.ANY)
testargs[-1] = 'SMAC4BO'
cli = SMACCLI()
with mock.patch("smac.smac_cli.SMAC4BO") as MSMAC:
MSMAC.return_value.optimize.return_value = True
cli.main_cli(testargs[1:])
MSMAC.assert_called_once_with(initial_configurations=None,
restore_incumbent=None,
run_id=2,
runhistory=None,
stats=None,
scenario=mock.ANY,
rng=mock.ANY)
testargs[-1] = 'SMAC4HPO'
cli = SMACCLI()
with mock.patch("smac.smac_cli.SMAC4HPO") as MSMAC:
MSMAC.return_value.optimize.return_value = True
cli.main_cli(testargs[1:])
MSMAC.assert_called_once_with(initial_configurations=None,
restore_incumbent=None,
run_id=2,
runhistory=None,
stats=None,
scenario=mock.ANY,
rng=mock.ANY)
testargs[-1] = 'Hydra'
cli = SMACCLI()
with mock.patch("smac.smac_cli.Hydra") as MSMAC:
MSMAC.return_value.optimize.return_value = True
cli.main_cli(testargs[1:])
MSMAC.assert_called_once_with(
initial_configurations=None,
restore_incumbent=None,
run_id=2,
incs_per_round=1,
n_iterations=3,
n_optimizers=1,
random_configuration_chooser=mock.ANY,
runhistory=None,
stats=None,
scenario=mock.ANY,
rng=mock.ANY,
val_set='train')
testargs[-1] = 'PSMAC'
cli = SMACCLI()
with mock.patch("smac.smac_cli.PSMAC") as MSMAC:
MSMAC.return_value.optimize.return_value = True
cli.main_cli(testargs[1:])
MSMAC.assert_called_once_with(run_id=2,
scenario=mock.ANY,
rng=mock.ANY,
n_incs=1,
n_optimizers=1,
shared_model=False,
validate=False)
class TestSMACCLI(unittest.TestCase):
def setUp(self):
base_directory = os.path.split(__file__)[0]
base_directory = os.path.abspath(
os.path.join(base_directory, '..', '..'))
self.current_dir = os.getcwd()
os.chdir(base_directory)
output_one_dir = "test/test_files/test_restore_state" # From scenario_one.txt
self.output_one = output_one_dir + "/run_1"
output_two_dir = "test/test_files/test_restored_state" # From scenario_two.txt
self.output_two = output_two_dir + "/run_1"
self.smaccli = SMACCLI()
self.scenario_one = "test/test_files/restore_scenario_one.txt"
self.scenario_two = "test/test_files/restore_scenario_two.txt"
self.output_dirs = [output_one_dir, output_two_dir]
def tearDown(self):
for output_dir in self.output_dirs:
if output_dir:
shutil.rmtree(output_dir, ignore_errors=True)
#pass
os.chdir(self.current_dir)
@attr('slow')
def test_run_and_restore(self):
"""
Testing basic restore functionality.
"""
# Run for 5 algo-calls
testargs = [
"python", "scripts/smac", "--scenario_file", self.scenario_one,
"--verbose", "DEBUG"
]
self.smaccli.main_cli(testargs[2:])
# Increase limit and run for 10 (so 5 more) by using restore_state
testargs = [
"python", "scripts/smac", "--restore_state", self.output_one,
"--scenario_file", self.scenario_two, "--verbose", "DEBUG"
]
self.smaccli.main_cli(testargs[2:])
def test_missing_dir(self):
"""
Testing error if dir is missing.
"""
testargs = [
"python", "scripts/smac", "--restore_state", "nonsense_test_dir",
"--scenario_file", self.scenario_two, "--verbose", "DEBUG"
]
self.assertRaises(FileNotFoundError,
lambda: self.smaccli.main_cli(testargs[2:]))
def test_illegal_input(self):
"""
Testing illegal input in smbo
"""
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter('test', 1, 10, 5))
scen = Scenario({'run_obj': 'quality', 'cs': cs})
stats = Stats(scen)
# Recorded runs but no incumbent.
stats.ta_runs = 10
smac = SMAC(scen, stats=stats, rng=np.random.RandomState(42))
self.output_dirs.append(scen.output_dir)
self.assertRaises(ValueError, smac.optimize)
# Incumbent but no recoreded runs.
incumbent = cs.get_default_configuration()
smac = SMAC(scen,
restore_incumbent=incumbent,
rng=np.random.RandomState(42))
self.assertRaises(ValueError, smac.optimize)
@attr('slow')
def test_same_dir(self):
"""
Testing possible error using same dir for restore
"""
# Run for 5 algo-calls
testargs = [
"python", "scripts/smac", "--scenario", self.scenario_one,
"--verbose", "DEBUG"
]
self.smaccli.main_cli(testargs[2:])
# Increase limit and run for 10 (so 5 more) by using restore_state
testargs = [
"python", "scripts/smac", "--restore_state", self.output_one,
"--scenario", self.scenario_two, "--verbose", "DEBUG"
]
# TODO: fix
try:
self.smaccli.main_cli(testargs[2:])
except FileNotFoundError:
pass
self.assertTrue(os.path.exists(self.output_one))
self.assertFalse(os.path.exists(self.output_one + '.OLD'))
self.assertTrue(os.path.exists(self.output_two))
self.assertFalse(os.path.exists(self.output_two + '.OLD'))
#!/usr/bin/env python
import logging
import sys
import os
import inspect
cmd_folder = os.path.realpath(
os.path.abspath(os.path.split(inspect.getfile(inspect.currentframe()))[0]))
cmd_folder = os.path.realpath(os.path.join(cmd_folder, ".."))
if cmd_folder not in sys.path:
sys.path.insert(0, cmd_folder)
from smac.smac_cli import SMACCLI
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
smac = SMACCLI()
smac.main_cli()