def test_build_from_args_debug_mode(script_path):
"""Try building experiment in debug mode"""
update_singletons()
experiment_builder.build_from_args({
'name':
'whatever',
'user_args': [script_path, '--mini-batch~uniform(32, 256)']
})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
experiment_builder.build_from_args({
'name':
'whatever',
'user_args': [script_path, '--mini-batch~uniform(32, 256)'],
'debug':
True
})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_build_from_args_debug_mode(script_path):
"""Try building experiment in debug mode"""
update_singletons()
experiment_builder.build_from_args(
{
"name": "whatever",
"user_args": [script_path, "--mini-batch~uniform(32, 256)"],
}
)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
experiment_builder.build_from_args(
{
"name": "whatever",
"user_args": [script_path, "--mini-batch~uniform(32, 256)"],
"debug": True,
}
)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def main(args):
"""Build experiment and execute hunt command"""
args["root"] = None
args["leafs"] = []
# TODO: simplify when parameter parsing is refactored
experiment = experiment_builder.build_from_args(args)
if args["init_only"]:
return
config = experiment_builder.get_cmd_config(args)
worker_config = orion.core.config.worker.to_dict()
if config.get("worker"):
worker_config.update(config.get("worker"))
# If EVC is not enabled, we force Consumer to ignore code changes.
if not config["branching"].get("enable", orion.core.config.evc.enable):
ignore_code_changes = True
else:
ignore_code_changes = config["branching"].get("ignore_code_changes")
with sigterm_as_interrupt():
workon(experiment,
ignore_code_changes=ignore_code_changes,
**worker_config)
def test_build_from_args_no_hit(config_file, random_dt, script_path, new_config):
"""Try building experiment when not in db"""
cmdargs = {
"name": "supernaekei",
"config": config_file,
"user_args": [script_path, "x~uniform(0,10)"],
}
with OrionState(experiments=[], trials=[]):
with pytest.raises(NoConfigurationError) as exc_info:
experiment_builder.get_from_args(cmdargs)
assert "No experiment with given name 'supernaekei' and version '*'" in str(
exc_info.value
)
exp = experiment_builder.build_from_args(cmdargs)
assert exp.name == cmdargs["name"]
assert exp.configuration["refers"] == {
"adapter": [],
"parent_id": None,
"root_id": exp._id,
}
assert exp.metadata["datetime"] == random_dt
assert exp.metadata["user"] == "dendi"
assert exp.metadata["user_script"] == cmdargs["user_args"][0]
assert exp.metadata["user_args"] == cmdargs["user_args"]
assert exp.max_trials == 100
assert exp.max_broken == 5
assert exp.algorithms.configuration == {"random": {"seed": None}}
def test_build_from_args_no_hit(config_file, random_dt, script_path,
new_config):
"""Try building experiment when not in db"""
cmdargs = {
'name': 'supernaekei',
'config': config_file,
'user_args': [script_path, 'x~uniform(0,10)']
}
with OrionState(experiments=[], trials=[]):
with pytest.raises(ValueError) as exc_info:
experiment_builder.build_view_from_args(cmdargs)
assert "No experiment with given name 'supernaekei' and version '*'" in str(
exc_info.value)
exp = experiment_builder.build_from_args(cmdargs)
assert exp.name == cmdargs['name']
assert exp.configuration['refers'] == {
'adapter': [],
'parent_id': None,
'root_id': exp._id
}
assert exp.metadata['datetime'] == random_dt
assert exp.metadata['user'] == 'dendi'
assert exp.metadata['user_script'] == cmdargs['user_args'][0]
assert exp.metadata['user_args'] == cmdargs['user_args']
assert exp.pool_size == 1
assert exp.max_trials == 100
assert exp.algorithms.configuration == {'random': {'seed': None}}
def test_build_from_args_force_user(new_config):
"""Try building experiment view when in db"""
cmdargs = {"name": new_config["name"]}
cmdargs["user"] = "******"
with OrionState(experiments=[new_config], trials=[]):
# Test that experiment already exists
exp_view = experiment_builder.build_from_args(cmdargs)
assert exp_view.metadata["user"] == "tsirif"
def test_build_from_args_force_user(new_config):
"""Try building experiment view when in db"""
cmdargs = {'name': new_config['name']}
cmdargs['user'] = '******'
with OrionState(experiments=[new_config], trials=[]):
# Test that experiment already exists
exp_view = experiment_builder.build_from_args(cmdargs)
assert exp_view.metadata['user'] == 'tsirif'
def test_workon():
"""Test scenario having a configured experiment already setup."""
name = "voici_voila"
config = {"name": name}
config["algorithms"] = {"gradient_descent": {"learning_rate": 0.1}}
config["pool_size"] = 1
config["max_trials"] = 100
config["exp_max_broken"] = 5
config["user_args"] = [
os.path.abspath(os.path.join(os.path.dirname(__file__),
"black_box.py")),
"-x~uniform(-50, 50, precision=None)",
]
with OrionState():
experiment = experiment_builder.build_from_args(config)
workon(experiment, 100, 100, 100, 100, 100)
storage = get_storage()
exp = list(storage.fetch_experiments({"name": name}))
assert len(exp) == 1
exp = exp[0]
assert "_id" in exp
assert exp["name"] == name
assert exp["pool_size"] == 1
assert exp["max_trials"] == 100
assert exp["max_broken"] == 5
assert exp["algorithms"] == {
"gradient_descent": {
"learning_rate": 0.1,
"dx_tolerance": 1e-7
}
}
assert "user" in exp["metadata"]
assert "datetime" in exp["metadata"]
assert "user_script" in exp["metadata"]
assert exp["metadata"]["user_args"] == config["user_args"]
trials = list(storage.fetch_trials(experiment))
assert len(trials) <= 15
trials = list(sorted(trials, key=lambda trial: trial.submit_time))
assert trials[-1].status == "completed"
for result in trials[-1].results:
assert result.type != "constraint"
if result.type == "objective":
assert abs(result.value - 23.4) < 1e-6
assert result.name == "example_objective"
elif result.type == "gradient":
res = numpy.asarray(result.value)
assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7
assert result.name == "example_gradient"
params = trials[-1].params
assert len(params) == 1
px = params["/x"]
assert isinstance(px, float)
assert (px - 34.56789) < 1e-5
def test_workon():
"""Test scenario having a configured experiment already setup."""
name = "voici_voila"
config = {"name": name}
config["algorithms"] = {"random": {"seed": 1}}
config["max_trials"] = 50
config["exp_max_broken"] = 5
config["user_args"] = [
os.path.abspath(os.path.join(os.path.dirname(__file__),
"black_box.py")),
"-x~uniform(-50, 50, precision=None)",
]
with OrionState():
experiment = experiment_builder.build_from_args(config)
workon(
experiment,
n_workers=2,
max_trials=10,
max_broken=5,
max_idle_time=20,
heartbeat=20,
user_script_config="config",
interrupt_signal_code=120,
ignore_code_changes=True,
executor="joblib",
executor_configuration={"backend": "threading"},
)
storage = get_storage()
exp = list(storage.fetch_experiments({"name": name}))
assert len(exp) == 1
exp = exp[0]
assert "_id" in exp
assert exp["name"] == name
assert exp["max_trials"] == 50
assert exp["max_broken"] == 5
assert exp["algorithms"] == {"random": {"seed": 1}}
assert "user" in exp["metadata"]
assert "datetime" in exp["metadata"]
assert "user_script" in exp["metadata"]
assert exp["metadata"]["user_args"] == config["user_args"]
trials = experiment.fetch_trials_by_status("completed")
assert len(trials) <= 22
trials = list(sorted(trials, key=lambda trial: trial.submit_time))
assert trials[-1].status == "completed"
params = trials[-1].params
assert len(params) == 1
px = params["/x"]
assert isinstance(px, float)
assert (px - 34.56789) < 20
def test_workon():
"""Test scenario having a configured experiment already setup."""
name = 'voici_voila'
config = {'name': name}
config['algorithms'] = {
'gradient_descent': {
'learning_rate': 0.1
}
}
config['pool_size'] = 1
config['max_trials'] = 100
config['user_args'] = [
os.path.abspath(os.path.join(os.path.dirname(__file__), "black_box.py")),
"-x~uniform(-50, 50, precision=None)"]
with OrionState():
experiment = experiment_builder.build_from_args(config)
workon(experiment, 100, 100, 100, 100, 100)
storage = get_storage()
exp = list(storage.fetch_experiments({'name': name}))
assert len(exp) == 1
exp = exp[0]
assert '_id' in exp
assert exp['name'] == name
assert exp['pool_size'] == 1
assert exp['max_trials'] == 100
assert exp['algorithms'] == {'gradient_descent': {'learning_rate': 0.1,
'dx_tolerance': 1e-7}}
assert 'user' in exp['metadata']
assert 'datetime' in exp['metadata']
assert 'user_script' in exp['metadata']
assert exp['metadata']['user_args'] == config['user_args']
trials = list(storage.fetch_trials(experiment))
assert len(trials) <= 15
trials = list(sorted(trials, key=lambda trial: trial.submit_time))
assert trials[-1].status == 'completed'
for result in trials[-1].results:
assert result.type != 'constraint'
if result.type == 'objective':
assert abs(result.value - 23.4) < 1e-6
assert result.name == 'example_objective'
elif result.type == 'gradient':
res = numpy.asarray(result.value)
assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7
assert result.name == 'example_gradient'
params = trials[-1]._params
assert len(params) == 1
assert params[0].name == '/x'
assert params[0].type == 'real'
assert (params[0].value - 34.56789) < 1e-5
def main(args):
"""Build experiment and execute hunt command"""
args['root'] = None
args['leafs'] = []
# TODO: simplify when parameter parsing is refactored
experiment = experiment_builder.build_from_args(args)
config = experiment_builder.get_cmd_config(args)
worker_config = orion.core.config.worker.to_dict()
if config.get('worker'):
worker_config.update(config.get('worker'))
workon(experiment, **worker_config)
def test_build_from_args_without_cmd(old_config_file, script_path, new_config):
"""Try building experiment without commandline when in db (no branch)"""
name = 'supernaekei'
cmdargs = {'name': name, 'config': old_config_file}
with OrionState(experiments=[new_config], trials=[]):
# Test that experiment already exists (this should fail otherwise)
experiment_builder.build_view_from_args(cmdargs)
exp = experiment_builder.build_from_args(cmdargs)
assert exp._id == new_config['_id']
assert exp.name == new_config['name']
assert exp.configuration['refers'] == new_config['refers']
assert exp.metadata == new_config['metadata']
assert exp.max_trials == new_config['max_trials']
assert exp.algorithms.configuration == new_config['algorithms']
def main(args):
"""Build experiment and execute hunt command"""
args["root"] = None
args["leafs"] = []
# TODO: simplify when parameter parsing is refactored
experiment = experiment_builder.build_from_args(args)
if args["init_only"]:
return
config = experiment_builder.get_cmd_config(args)
worker_config = orion.core.config.worker.to_dict()
if config.get("worker"):
worker_config.update(config.get("worker"))
workon(experiment,
ignore_code_changes=config["branching"].get("ignore_code_changes"),
**worker_config)
def test_build_from_args_without_cmd(old_config_file, script_path, new_config):
"""Try building experiment without commandline when in db (no branch)"""
name = "supernaekei"
cmdargs = {"name": name, "config": old_config_file}
with OrionState(experiments=[new_config], trials=[]):
# Test that experiment already exists (this should fail otherwise)
experiment_builder.get_from_args(cmdargs)
exp = experiment_builder.build_from_args(cmdargs)
assert exp._id == new_config["_id"]
assert exp.name == new_config["name"]
assert exp.configuration["refers"] == new_config["refers"]
assert exp.metadata == new_config["metadata"]
assert exp.max_trials == new_config["max_trials"]
assert exp.max_broken == new_config["max_broken"]
assert exp.algorithms.configuration == new_config["algorithms"]
def test_build_from_args_hit(old_config_file, script_path, new_config):
"""Try building experiment when in db (no branch)"""
cmdargs = {
'name': 'supernaekei',
'config': old_config_file,
'user_args':
[script_path, '--mini-batch~uniform(32, 256, discrete=True)']
}
with OrionState(experiments=[new_config], trials=[]):
# Test that experiment already exists
experiment_builder.build_view_from_args(cmdargs)
exp = experiment_builder.build_from_args(cmdargs)
assert exp._id == new_config['_id']
assert exp.name == new_config['name']
assert exp.version == 1
assert exp.configuration['refers'] == new_config['refers']
assert exp.metadata == new_config['metadata']
assert exp.max_trials == new_config['max_trials']
assert exp.algorithms.configuration == new_config['algorithms']
def test_build_from_args_hit(old_config_file, script_path, new_config):
"""Try building experiment when in db (no branch)"""
cmdargs = {
"name": "supernaekei",
"config": old_config_file,
"user_args": [script_path, "--mini-batch~uniform(32, 256, discrete=True)"],
}
with OrionState(experiments=[new_config], trials=[]):
# Test that experiment already exists
experiment_builder.get_from_args(cmdargs)
exp = experiment_builder.build_from_args(cmdargs)
assert exp._id == new_config["_id"]
assert exp.name == new_config["name"]
assert exp.version == 1
assert exp.configuration["refers"] == new_config["refers"]
assert exp.metadata == new_config["metadata"]
assert exp.max_trials == new_config["max_trials"]
assert exp.max_broken == new_config["max_broken"]
assert exp.algorithms.configuration == new_config["algorithms"]
def main(args):
"""Build and initialize experiment"""
# By building the experiment, we create a new experiment document in database
log.warning("Command init_only is deprecated and will be removed in v0.3. "
"Use orion hunt --init-only instead.")
experiment_builder.build_from_args(args)
def main(args):
"""Build and initialize experiment"""
# By building the experiment, we create a new experiment document in database
experiment_builder.build_from_args(args)
