def test_build_from_hit(old_config_file, create_db_instance, exp_config):
"""Try building experiment when in db (no branch)"""
cmdargs = {
'name':
'supernaedo2',
'config':
old_config_file,
'user_args': [
'full_path/main.py',
"--encoding_layer~choices(['rnn', 'lstm', 'gru'])",
"--decoding_layer~choices(['rnn', 'lstm_with_attention', 'gru'])"
]
}
# Test that experiment already exists
ExperimentBuilder().build_view_from(cmdargs)
exp = ExperimentBuilder().build_from(cmdargs)
assert exp._init_done is True
assert exp._db is create_db_instance
assert exp._id == exp_config[0][0]['_id']
assert exp.name == exp_config[0][0]['name']
assert exp.configuration['refers'] == exp_config[0][0]['refers']
assert exp.metadata == exp_config[0][0]['metadata']
assert exp._last_fetched == exp_config[0][0]['metadata']['datetime']
assert exp.pool_size == exp_config[0][0]['pool_size']
assert exp.max_trials == exp_config[0][0]['max_trials']
assert exp.algorithms.configuration == exp_config[0][0]['algorithms']
def test_build_from_config_no_hit(config_file, create_db_instance, exp_config, random_dt,
script_path):
"""Try building experiment from config when not in db"""
cmdargs = {'name': 'supernaekei', 'config': config_file,
'user_args': [script_path,
'-x~uniform(0,10)']}
with pytest.raises(ValueError) as exc_info:
ExperimentBuilder().build_view_from(cmdargs)
assert "No experiment with given name 'supernaekei' for user 'tsirif'" in str(exc_info.value)
full_config = ExperimentBuilder().fetch_full_config(cmdargs)
exp = ExperimentBuilder().build_from_config(full_config)
assert exp._init_done is True
assert get_db(exp) is create_db_instance
assert exp.name == cmdargs['name']
assert exp.configuration['refers'] == {'adapter': [], 'parent_id': None, 'root_id': exp._id}
assert exp.metadata['datetime'] == random_dt
assert exp.metadata['user'] == 'tsirif'
assert exp.metadata['user_script'] == cmdargs['user_args'][0]
assert exp.metadata['user_args'] == cmdargs['user_args'][1:]
assert exp.pool_size == 1
assert exp.max_trials == 100
assert not exp.is_done
assert exp.algorithms.configuration == {'random': {'seed': None}}
def test_fetch_full_config_old_config(old_config_file, exp_config, random_dt):
"""Verify full config with old config (not causing branch)"""
cmdargs = {
'name':
'supernaedo2',
'config':
old_config_file,
'user_args': [
'full_path/main.py',
"--encoding_layer~choices(['rnn', 'lstm', 'gru'])",
"--decoding_layer~choices(['rnn', 'lstm_with_attention', 'gru'])"
]
}
full_config = ExperimentBuilder().fetch_full_config(cmdargs)
cmdconfig = ExperimentBuilder().fetch_file_config(cmdargs)
full_config['metadata']['orion_version'] = exp_config[0][0]['metadata'][
'orion_version']
assert full_config['name'] == exp_config[0][0]['name']
assert full_config['refers'] == exp_config[0][0]['refers']
assert full_config['metadata'] == exp_config[0][0]['metadata']
assert full_config['pool_size'] == cmdconfig['pool_size']
assert full_config['max_trials'] == cmdconfig['max_trials']
assert full_config['algorithms'] == cmdconfig['algorithms']
def test_build_from_config_no_commandline_config(config_file):
"""Try building experiment with no commandline configuration."""
cmdargs = {'name': 'supernaekei', 'config': config_file}
full_config = ExperimentBuilder().fetch_full_config(cmdargs)
with pytest.raises(NoConfigurationError):
ExperimentBuilder().build_from_config(full_config)
def test_build_without_config_hit(old_config_file, create_db_instance, exp_config, script_path):
"""Try building experiment without commandline config when in db (no branch)"""
cmdargs = {'name': 'supernaedo2',
'config': old_config_file,
'user_args': [script_path,
"--encoding_layer~choices(['rnn', 'lstm', 'gru'])",
"--decoding_layer~choices(['rnn', 'lstm_with_attention', 'gru'])"]}
# Test that experiment already exists
ExperimentBuilder().build_view_from(cmdargs)
cmdargs = {'name': 'supernaedo2'}
exp_view = ExperimentBuilder().build_view_from(cmdargs)
exp = ExperimentBuilder().build_from_config(exp_view.configuration)
assert exp._init_done is True
assert get_db(exp) is create_db_instance
assert exp._id == exp_config[0][0]['_id']
assert exp.name == exp_config[0][0]['name']
assert exp.configuration['refers'] == exp_config[0][0]['refers']
assert exp.metadata == exp_config[0][0]['metadata']
assert exp.pool_size == exp_config[0][0]['pool_size']
assert exp.max_trials == exp_config[0][0]['max_trials']
assert exp.algorithms.configuration == exp_config[0][0]['algorithms']
def test_build_view_from_force_user(config_file, create_db_instance, exp_config, random_dt):
"""Try building experiment view when in db"""
# Verify default behavior properly fetches bouthilx
assert getpass.getuser() == 'bouthilx'
cmdargs = {'name': 'supernaedo2', 'config': config_file}
with pytest.raises(ValueError) as exc_info:
exp_view = ExperimentBuilder().build_view_from(cmdargs)
assert "No experiment with given name 'supernaedo2' for user 'bouthilx'" in str(exc_info.value)
cmdargs['user'] = '******'
exp_view = ExperimentBuilder().build_view_from(cmdargs)
assert exp_view.metadata['user'] == 'tsirif'
def main(args):
"""Fetch config and insert new point"""
command_line_user_args = args.pop('user_args', [None])[1:]
experiment_view = ExperimentBuilder().build_view_from(args)
transformed_args = _build_from(command_line_user_args)
exp_space = experiment_view.space
values = _create_tuple_from_values(transformed_args, exp_space)
trial = tuple_to_trial(values, exp_space)
ExperimentBuilder().build_from_config(
experiment_view.configuration).register_trials([trial])
def main(args):
"""List all experiments inside database."""
builder = ExperimentBuilder()
config = builder.fetch_full_config(args, use_db=False)
builder.setup_storage(config)
query = {}
if args['name']:
query['name'] = args['name']
experiments = get_storage().fetch_experiments(query)
if args['name']:
root_experiments = experiments
else:
root_experiments = [
exp for exp in experiments
if exp['refers'].get('root_id', exp['_id']) == exp['_id']
]
for root_experiment in root_experiments:
root = EVCBuilder().build_view_from({
'name':
root_experiment['name'],
'version':
root_experiment.get('version')
}).node
print_tree(root, nameattr='tree_name')
def main(args):
"""Fetch config and status experiments"""
builder = ExperimentBuilder()
local_config = builder.fetch_full_config(args, use_db=False)
builder.setup_storage(local_config)
args['all_trials'] = args.pop('all', False)
experiments = get_experiments(args)
if not experiments:
print("No experiment found")
return
if args.get('name'):
print_evc([experiments[0]], **args)
return
if args.get('version'):
if args.get('collapse') or args.get('expand_versions'):
raise RuntimeError(
"Cannot fetch specific version of experiments with --collapse "
"or --expand-versions.")
print_evc(filter(lambda e: e.refers.get('parent_id') is None, experiments),
**args)
def main(args):
"""List all experiments inside database."""
builder = ExperimentBuilder()
config = builder.fetch_full_config(args, use_db=False)
builder.setup_database(config)
query = {}
if args['name']:
query['name'] = args['name']
experiments = Database().read("experiments", query)
if args['name']:
root_experiments = experiments
else:
root_experiments = [
exp for exp in experiments
if exp['refers'].get('root_id', exp['_id']) == exp['_id']
]
for root_experiment in root_experiments:
root = EVCBuilder().build_view_from({
'name': root_experiment['name']
}).node
print_tree(root)
def main(args):
"""Upgrade the databases for current version"""
print(
"Upgrading your database may damage your data. Make sure to make a backup before the "
"upgrade and stop any other process that may read/write the database during the upgrade."
)
if not args.get('force'):
action = ''
while action not in ['y', 'yes', 'no', 'n']:
action = ask_question("Do you wish to proceed? (y/N)", "N").lower()
if action in ['no', 'n']:
sys.exit(0)
experiment_builder = ExperimentBuilder()
local_config = experiment_builder.fetch_full_config(args, use_db=False)
local_config['protocol'] = {'type': 'legacy', 'setup': False}
experiment_builder.setup_storage(local_config)
storage = get_storage()
upgrade_db_specifics(storage)
print('Updating documents...')
upgrade_documents(storage)
print('Database upgrade completed successfully')
def test_build_view_from_no_hit(config_file, create_db_instance, exp_config):
"""Try building experiment view when not in db"""
cmdargs = {'name': 'supernaekei', 'config': config_file}
with pytest.raises(ValueError) as exc_info:
ExperimentBuilder().build_view_from(cmdargs)
assert "No experiment with given name 'supernaekei' for user 'tsirif'" in str(exc_info.value)
def main():
""" This function aims at fetching the results of an ORION Experiment.
An `experiment_name` needs to be provided. Also, it is possible to
retrieve results from complete trials.
"""
args = manage_arguments()
# some_datetime = datetime.datetime.now() - datetime.timedelta(minutes=5)
experiment = ExperimentBuilder().build_view_from(
{"name": args.experiment_name})
pprint.pprint(experiment.stats)
query = {}
if args.only_completed:
query['status'] = 'completed'
# query['end_time'] = {'$gte': some_datetime}
for trial in experiment.fetch_trials(query):
print(trial.id)
print(trial.status)
print(trial.params)
print(trial.results)
print()
pprint.pprint(trial.to_dict())
def main(args):
"""Fetch config and insert new point"""
command_line_user_args = args.pop('user_args', [None])[1:]
# TODO: Views are not fully configured until configuration is refactored
experiment = ExperimentBuilder().build_view_from(args)
# TODO: Remove this line when views gets fully configured
experiment = ExperimentBuilder().build_from(args)
transformed_args = _build_from(command_line_user_args)
exp_space = experiment.space
values = _create_tuple_from_values(transformed_args, exp_space)
trial = tuple_to_trial(values, exp_space)
experiment.register_trial(trial)
def test_build_from_force_user(old_config_file, create_db_instance, exp_config,
random_dt):
"""Try building experiment view when in db"""
cmdargs = {'name': 'supernaedo2', 'config': old_config_file}
cmdargs['user'] = '******'
exp_view = ExperimentBuilder().build_from(cmdargs)
assert exp_view.metadata['user'] == 'tsirif'
def insert_trials(experiment_name, points, cmdconfig=None, raise_exc=True):
"""Insert sets of parameters manually, defined in `points`, as new trials
for the experiment name, `experiment_name`.
:param experiment_name: Name of the experiment which the new trials are
going to be associated with
:param points: list of tuples in agreement with experiment's parameter space
:param raise_exc: whether an inappropriate tuple of parameters will raise
an exception or it will be ignored
.. note:: If `raise_exc` is True, no set of parameters will be inserted. If
it is False, only the valid ones will be inserted; the rest will be ignored.
.. note:: This cannot be used to prepopulate a future experiment. So,
an experiment with `experiment_name` should already be configured in
the database.
"""
cmdconfig = cmdconfig if cmdconfig else {}
cmdconfig['name'] = experiment_name
experiment_view = ExperimentBuilder().build_view_from(
{'config': cmdconfig})
valid_points = []
print(experiment_view.space)
for point in points:
try:
assert point in experiment_view.space
valid_points.append(point)
except AssertionError:
if raise_exc:
raise
if not valid_points:
return
new_trials = list(
map(
lambda data: format_trials.tuple_to_trial(data, experiment_view.
space), valid_points))
for new_trial in new_trials:
ExperimentBuilder().build_from(
experiment_view.configuration).register_trial(new_trial)
def main(args):
"""Build experiment and execute hunt command"""
args['root'] = None
args['leafs'] = []
# TODO: simplify when parameter parsing is refactored
worker_trials = ExperimentBuilder().fetch_full_config(args)['worker_trials']
experiment = EVCBuilder().build_from(args)
workon(experiment, worker_trials)
def build_storage():
"""Build storage from scratch"""
null_db_instances()
experiment_builder = ExperimentBuilder()
local_config = experiment_builder.fetch_full_config({}, use_db=False)
experiment_builder.setup_storage(local_config)
return get_storage()
def family_with_trials(two_experiments):
"""Create two related experiments with all types of trials."""
exp = ExperimentBuilder().build_from({'name': 'test_double_exp'})
exp2 = ExperimentBuilder().build_from({'name': 'test_double_exp_child'})
x = {'name': '/x', 'type': 'real'}
y = {'name': '/y', 'type': 'real'}
x_value = 0
for status in Trial.allowed_stati:
x['value'] = x_value
y['value'] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x['value'] = x_value
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
x_value += 1
Database().write('trials', trial.to_dict())
Database().write('trials', trial2.to_dict())
def single_with_trials(single_without_success):
"""Create an experiment with all types of trials."""
exp = ExperimentBuilder().build_from({'name': 'test_single_exp'})
x = {'name': '/x', 'type': 'real', 'value': 100}
results = {"name": "obj", "type": "objective", "value": 0}
trial = Trial(experiment=exp.id, params=[x], status='completed', results=[results])
Database().write('trials', trial.to_dict())
def test_fetch_config_from_db_hit(config_file, exp_config, random_dt):
"""Verify db config when experiment is in db"""
cmdargs = {'name': 'supernaedo2', 'config': config_file}
db_config = ExperimentBuilder().fetch_config_from_db(cmdargs)
assert db_config['name'] == exp_config[0][0]['name']
assert db_config['refers'] == exp_config[0][0]['refers']
assert db_config['metadata'] == exp_config[0][0]['metadata']
assert db_config['pool_size'] == exp_config[0][0]['pool_size']
assert db_config['max_trials'] == exp_config[0][0]['max_trials']
assert db_config['algorithms'] == exp_config[0][0]['algorithms']
def test_fetch_local_config(config_file):
"""Test local config (default, env_vars, cmdconfig, cmdargs)"""
cmdargs = {"config": config_file}
local_config = ExperimentBuilder().fetch_full_config(cmdargs, use_db=False)
assert local_config['algorithms'] == 'random'
assert local_config['database']['host'] == 'mongodb://*****:*****@localhost'
assert local_config['database']['name'] == 'orion_test'
assert local_config['database']['type'] == 'mongodb'
assert local_config['max_trials'] == 100
assert local_config['name'] == 'voila_voici'
assert local_config['pool_size'] == 1
def test_build_view_from(config_file, create_db_instance, exp_config, random_dt):
"""Try building experiment view when in db"""
cmdargs = {'name': 'supernaedo2', 'config': config_file}
exp_view = ExperimentBuilder().build_view_from(cmdargs)
assert exp_view._experiment._init_done is False
assert get_view_db(exp_view) is create_db_instance
assert exp_view._id == exp_config[0][0]['_id']
assert exp_view.name == exp_config[0][0]['name']
assert exp_view.configuration['refers'] == exp_config[0][0]['refers']
assert exp_view.metadata == exp_config[0][0]['metadata']
assert exp_view.pool_size == exp_config[0][0]['pool_size']
assert exp_view.max_trials == exp_config[0][0]['max_trials']
def test_trial_working_dir_is_changed(config, monkeypatch):
"""Check that trial has its working_dir attribute changed."""
exp = ExperimentBuilder().build_from(config)
trial = tuple_to_trial((1.0, ), exp.space)
exp.register_trial(trial)
con = Consumer(exp)
con.consume(trial)
assert trial.working_dir is not None
assert trial.working_dir == con.working_dir + "/exp_" + trial.id
def test_fetch_full_config_no_hit(config_file, exp_config, random_dt):
"""Verify full config when experiment not in db"""
cmdargs = {'name': 'supernaekei', 'config': config_file}
full_config = ExperimentBuilder().fetch_full_config(cmdargs)
assert full_config['name'] == 'supernaekei'
assert full_config['algorithms'] == 'random'
assert full_config['max_trials'] == 100
assert full_config['name'] == 'supernaekei'
assert full_config['pool_size'] == 1
assert full_config['metadata']['user'] == 'tsirif'
assert 'datetime' not in full_config['metadata']
assert 'refers' not in full_config
def three_family_with_trials(three_experiments_family, family_with_trials):
"""Create three experiments, all related, two direct children, with all types of trials."""
exp = ExperimentBuilder().build_from({'name': 'test_double_exp_child2'})
x = {'name': '/x', 'type': 'real'}
z = {'name': '/z', 'type': 'real'}
x_value = 0
for status in Trial.allowed_stati:
x['value'] = x_value
z['value'] = x_value * 100
trial = Trial(experiment=exp.id, params=[x, z], status=status)
x_value += 1
Database().write('trials', trial.to_dict())
def one_experiment_changed_vcs(one_experiment):
"""Create an experiment without trials."""
experiment = ExperimentBuilder().build_from(
{'name': one_experiment['name']})
experiment.metadata['VCS'] = {
'type': 'git',
'is_dirty': False,
'HEAD_sha': 'new',
'active_branch': 'master',
'diff_sha': None
}
get_storage().update_experiment(experiment, metadata=experiment.metadata)
def test_tpe_optimizer_actually_optimize(database, monkeypatch):
"""Check if TPE Optimizer has better optimization than random search."""
monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__)))
best_random_search = 23.403275057472825
orion.core.cli.main(["hunt", "--max-trials", "40", "--config",
"./orion_config_tpe.yaml", "./black_box.py",
"-x~uniform(-50, 50)"])
with open("./orion_config_tpe.yaml", "r") as f:
exp = ExperimentBuilder().build_view_from({'config': f})
objective = exp.stats['best_evaluation']
assert best_random_search > objective
def single_without_success(one_experiment):
"""Create an experiment without a succesful trial."""
statuses = list(Trial.allowed_stati)
statuses.remove('completed')
exp = ExperimentBuilder().build_from({'name': 'test_single_exp'})
x = {'name': '/x', 'type': 'real'}
x_value = 0
for status in statuses:
x['value'] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x_value += 1
Database().write('trials', trial.to_dict())
def test_resilience(monkeypatch):
"""Test if Oríon stops after enough broken trials."""
monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__)))
MAX_BROKEN = 3
orion.core.config.worker.max_broken = 3
orion.core.cli.main([
"hunt", "--config", "./orion_config_random.yaml", "./broken_box.py",
"-x~uniform(-50, 50)"
])
exp = ExperimentBuilder().build_from({'name': 'demo_random_search'})
assert len(exp.fetch_trials_by_status('broken')) == MAX_BROKEN
