def main(args): """Upgrade the databases for current version""" print( "Upgrading your database may damage your data. Make sure to make a backup before the " "upgrade and stop any other process that may read/write the database during the upgrade." ) if not args.get('force'): action = '' while action not in ['y', 'yes', 'no', 'n']: action = ask_question("Do you wish to proceed? (y/N)", "N").lower() if action in ['no', 'n']: sys.exit(0) experiment_builder = ExperimentBuilder() local_config = experiment_builder.fetch_full_config(args, use_db=False) local_config['protocol'] = {'type': 'legacy', 'setup': False} experiment_builder.setup_storage(local_config) storage = get_storage() upgrade_db_specifics(storage) print('Updating documents...') upgrade_documents(storage) print('Database upgrade completed successfully')
def main(): """ This function aims at fetching the results of an ORION Experiment. An `experiment_name` needs to be provided. Also, it is possible to retrieve results from complete trials. """ args = manage_arguments() # some_datetime = datetime.datetime.now() - datetime.timedelta(minutes=5) experiment = ExperimentBuilder().build_view_from( {"name": args.experiment_name}) pprint.pprint(experiment.stats) query = {} if args.only_completed: query['status'] = 'completed' # query['end_time'] = {'$gte': some_datetime} for trial in experiment.fetch_trials(query): print(trial.id) print(trial.status) print(trial.params) print(trial.results) print() pprint.pprint(trial.to_dict())
def main(args): """List all experiments inside database.""" builder = ExperimentBuilder() config = builder.fetch_full_config(args, use_db=False) builder.setup_database(config) query = {} if args['name']: query['name'] = args['name'] experiments = Database().read("experiments", query) if args['name']: root_experiments = experiments else: root_experiments = [ exp for exp in experiments if exp['refers'].get('root_id', exp['_id']) == exp['_id'] ] for root_experiment in root_experiments: root = EVCBuilder().build_view_from({ 'name': root_experiment['name'] }).node print_tree(root)
def test_build_from_config_no_hit(config_file, create_db_instance, exp_config, random_dt, script_path): """Try building experiment from config when not in db""" cmdargs = {'name': 'supernaekei', 'config': config_file, 'user_args': [script_path, '-x~uniform(0,10)']} with pytest.raises(ValueError) as exc_info: ExperimentBuilder().build_view_from(cmdargs) assert "No experiment with given name 'supernaekei' for user 'tsirif'" in str(exc_info.value) full_config = ExperimentBuilder().fetch_full_config(cmdargs) exp = ExperimentBuilder().build_from_config(full_config) assert exp._init_done is True assert get_db(exp) is create_db_instance assert exp.name == cmdargs['name'] assert exp.configuration['refers'] == {'adapter': [], 'parent_id': None, 'root_id': exp._id} assert exp.metadata['datetime'] == random_dt assert exp.metadata['user'] == 'tsirif' assert exp.metadata['user_script'] == cmdargs['user_args'][0] assert exp.metadata['user_args'] == cmdargs['user_args'][1:] assert exp.pool_size == 1 assert exp.max_trials == 100 assert not exp.is_done assert exp.algorithms.configuration == {'random': {'seed': None}}
def main(args): """List all experiments inside database.""" builder = ExperimentBuilder() config = builder.fetch_full_config(args, use_db=False) builder.setup_storage(config) query = {} if args['name']: query['name'] = args['name'] experiments = get_storage().fetch_experiments(query) if args['name']: root_experiments = experiments else: root_experiments = [ exp for exp in experiments if exp['refers'].get('root_id', exp['_id']) == exp['_id'] ] for root_experiment in root_experiments: root = EVCBuilder().build_view_from({ 'name': root_experiment['name'], 'version': root_experiment.get('version') }).node print_tree(root, nameattr='tree_name')
def main(args): """Fetch config and status experiments""" builder = ExperimentBuilder() local_config = builder.fetch_full_config(args, use_db=False) builder.setup_storage(local_config) args['all_trials'] = args.pop('all', False) experiments = get_experiments(args) if not experiments: print("No experiment found") return if args.get('name'): print_evc([experiments[0]], **args) return if args.get('version'): if args.get('collapse') or args.get('expand_versions'): raise RuntimeError( "Cannot fetch specific version of experiments with --collapse " "or --expand-versions.") print_evc(filter(lambda e: e.refers.get('parent_id') is None, experiments), **args)
def test_build_without_config_hit(old_config_file, create_db_instance, exp_config, script_path): """Try building experiment without commandline config when in db (no branch)""" cmdargs = {'name': 'supernaedo2', 'config': old_config_file, 'user_args': [script_path, "--encoding_layer~choices(['rnn', 'lstm', 'gru'])", "--decoding_layer~choices(['rnn', 'lstm_with_attention', 'gru'])"]} # Test that experiment already exists ExperimentBuilder().build_view_from(cmdargs) cmdargs = {'name': 'supernaedo2'} exp_view = ExperimentBuilder().build_view_from(cmdargs) exp = ExperimentBuilder().build_from_config(exp_view.configuration) assert exp._init_done is True assert get_db(exp) is create_db_instance assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] assert exp.algorithms.configuration == exp_config[0][0]['algorithms']
def test_build_from_config_no_commandline_config(config_file): """Try building experiment with no commandline configuration.""" cmdargs = {'name': 'supernaekei', 'config': config_file} full_config = ExperimentBuilder().fetch_full_config(cmdargs) with pytest.raises(NoConfigurationError): ExperimentBuilder().build_from_config(full_config)
def test_build_from_hit(old_config_file, create_db_instance, exp_config): """Try building experiment when in db (no branch)""" cmdargs = { 'name': 'supernaedo2', 'config': old_config_file, 'user_args': [ 'full_path/main.py', "--encoding_layer~choices(['rnn', 'lstm', 'gru'])", "--decoding_layer~choices(['rnn', 'lstm_with_attention', 'gru'])" ] } # Test that experiment already exists ExperimentBuilder().build_view_from(cmdargs) exp = ExperimentBuilder().build_from(cmdargs) assert exp._init_done is True assert exp._db is create_db_instance assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] assert exp._last_fetched == exp_config[0][0]['metadata']['datetime'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] assert exp.algorithms.configuration == exp_config[0][0]['algorithms']
def test_fetch_full_config_old_config(old_config_file, exp_config, random_dt): """Verify full config with old config (not causing branch)""" cmdargs = { 'name': 'supernaedo2', 'config': old_config_file, 'user_args': [ 'full_path/main.py', "--encoding_layer~choices(['rnn', 'lstm', 'gru'])", "--decoding_layer~choices(['rnn', 'lstm_with_attention', 'gru'])" ] } full_config = ExperimentBuilder().fetch_full_config(cmdargs) cmdconfig = ExperimentBuilder().fetch_file_config(cmdargs) full_config['metadata']['orion_version'] = exp_config[0][0]['metadata'][ 'orion_version'] assert full_config['name'] == exp_config[0][0]['name'] assert full_config['refers'] == exp_config[0][0]['refers'] assert full_config['metadata'] == exp_config[0][0]['metadata'] assert full_config['pool_size'] == cmdconfig['pool_size'] assert full_config['max_trials'] == cmdconfig['max_trials'] assert full_config['algorithms'] == cmdconfig['algorithms']
def build_storage(): """Build storage from scratch""" null_db_instances() experiment_builder = ExperimentBuilder() local_config = experiment_builder.fetch_full_config({}, use_db=False) experiment_builder.setup_storage(local_config) return get_storage()
def test_build_view_from_force_user(config_file, create_db_instance, exp_config, random_dt): """Try building experiment view when in db""" # Verify default behavior properly fetches bouthilx assert getpass.getuser() == 'bouthilx' cmdargs = {'name': 'supernaedo2', 'config': config_file} with pytest.raises(ValueError) as exc_info: exp_view = ExperimentBuilder().build_view_from(cmdargs) assert "No experiment with given name 'supernaedo2' for user 'bouthilx'" in str(exc_info.value) cmdargs['user'] = '******' exp_view = ExperimentBuilder().build_view_from(cmdargs) assert exp_view.metadata['user'] == 'tsirif'
def test_trial_working_dir_is_changed(config, monkeypatch): """Check that trial has its working_dir attribute changed.""" exp = ExperimentBuilder().build_from(config) trial = tuple_to_trial((1.0, ), exp.space) exp.register_trial(trial) con = Consumer(exp) con.consume(trial) assert trial.working_dir is not None assert trial.working_dir == con.working_dir + "/exp_" + trial.id
def main(args): """Fetch config and insert new point""" command_line_user_args = args.pop('user_args', [None])[1:] experiment_view = ExperimentBuilder().build_view_from(args) transformed_args = _build_from(command_line_user_args) exp_space = experiment_view.space values = _create_tuple_from_values(transformed_args, exp_space) trial = tuple_to_trial(values, exp_space) ExperimentBuilder().build_from_config( experiment_view.configuration).register_trials([trial])
def test_resilience(monkeypatch): """Test if Oríon stops after enough broken trials.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) MAX_BROKEN = 3 orion.core.config.worker.max_broken = 3 orion.core.cli.main([ "hunt", "--config", "./orion_config_random.yaml", "./broken_box.py", "-x~uniform(-50, 50)" ]) exp = ExperimentBuilder().build_from({'name': 'demo_random_search'}) assert len(exp.fetch_trials_by_status('broken')) == MAX_BROKEN
def one_experiment_changed_vcs(one_experiment): """Create an experiment without trials.""" experiment = ExperimentBuilder().build_from( {'name': one_experiment['name']}) experiment.metadata['VCS'] = { 'type': 'git', 'is_dirty': False, 'HEAD_sha': 'new', 'active_branch': 'master', 'diff_sha': None } get_storage().update_experiment(experiment, metadata=experiment.metadata)
def test_build_view_from_no_hit(config_file, create_db_instance, exp_config): """Try building experiment view when not in db""" cmdargs = {'name': 'supernaekei', 'config': config_file} with pytest.raises(ValueError) as exc_info: ExperimentBuilder().build_view_from(cmdargs) assert "No experiment with given name 'supernaekei' for user 'tsirif'" in str(exc_info.value)
def test_build_from_force_user(old_config_file, create_db_instance, exp_config, random_dt): """Try building experiment view when in db""" cmdargs = {'name': 'supernaedo2', 'config': old_config_file} cmdargs['user'] = '******' exp_view = ExperimentBuilder().build_from(cmdargs) assert exp_view.metadata['user'] == 'tsirif'
def insert_trials(experiment_name, points, cmdconfig=None, raise_exc=True): """Insert sets of parameters manually, defined in `points`, as new trials for the experiment name, `experiment_name`. :param experiment_name: Name of the experiment which the new trials are going to be associated with :param points: list of tuples in agreement with experiment's parameter space :param raise_exc: whether an inappropriate tuple of parameters will raise an exception or it will be ignored .. note:: If `raise_exc` is True, no set of parameters will be inserted. If it is False, only the valid ones will be inserted; the rest will be ignored. .. note:: This cannot be used to prepopulate a future experiment. So, an experiment with `experiment_name` should already be configured in the database. """ cmdconfig = cmdconfig if cmdconfig else {} cmdconfig['name'] = experiment_name experiment_view = ExperimentBuilder().build_view_from( {'config': cmdconfig}) valid_points = [] print(experiment_view.space) for point in points: try: assert point in experiment_view.space valid_points.append(point) except AssertionError: if raise_exc: raise if not valid_points: return new_trials = list( map( lambda data: format_trials.tuple_to_trial(data, experiment_view. space), valid_points)) for new_trial in new_trials: ExperimentBuilder().build_from( experiment_view.configuration).register_trial(new_trial)
def single_with_trials(single_without_success): """Create an experiment with all types of trials.""" exp = ExperimentBuilder().build_from({'name': 'test_single_exp'}) x = {'name': '/x', 'type': 'real', 'value': 100} results = {"name": "obj", "type": "objective", "value": 0} trial = Trial(experiment=exp.id, params=[x], status='completed', results=[results]) Database().write('trials', trial.to_dict())
def family_with_trials(two_experiments): """Create two related experiments with all types of trials.""" exp = ExperimentBuilder().build_from({'name': 'test_double_exp'}) exp2 = ExperimentBuilder().build_from({'name': 'test_double_exp_child'}) x = {'name': '/x', 'type': 'real'} y = {'name': '/y', 'type': 'real'} x_value = 0 for status in Trial.allowed_stati: x['value'] = x_value y['value'] = x_value trial = Trial(experiment=exp.id, params=[x], status=status) x['value'] = x_value trial2 = Trial(experiment=exp2.id, params=[x, y], status=status) x_value += 1 Database().write('trials', trial.to_dict()) Database().write('trials', trial2.to_dict())
def main(args): """Build experiment and execute hunt command""" args['root'] = None args['leafs'] = [] # TODO: simplify when parameter parsing is refactored worker_trials = ExperimentBuilder().fetch_full_config(args)['worker_trials'] experiment = EVCBuilder().build_from(args) workon(experiment, worker_trials)
def test_pacemaker_termination(config, monkeypatch): """Check if pacemaker stops as soon as the trial completes.""" exp = ExperimentBuilder().build_from(config) trial = tuple_to_trial((1.0, ), exp.space) exp.register_trial(trial) con = Consumer(exp) start = time.time() con.consume(trial) con.pacemaker.join() duration = time.time() - start assert duration < con.pacemaker.wait_time
def main(args): """Fetch config and status experiments""" builder = ExperimentBuilder() local_config = builder.fetch_full_config(args, use_db=False) builder.setup_database(local_config) experiments = get_experiments(args) if args.get('name'): print_status(experiments[0], all_trials=args.get('all'), collapse=args.get('collapse')) return for exp in filter(lambda e: e.refers.get('parent_id') is None, experiments): if args.get('collapse'): print_status(exp, all_trials=args.get('all'), collapse=True) else: print_status_recursively(exp, all_trials=args.get('all'))
def test_fetch_config_from_db_hit(config_file, exp_config, random_dt): """Verify db config when experiment is in db""" cmdargs = {'name': 'supernaedo2', 'config': config_file} db_config = ExperimentBuilder().fetch_config_from_db(cmdargs) assert db_config['name'] == exp_config[0][0]['name'] assert db_config['refers'] == exp_config[0][0]['refers'] assert db_config['metadata'] == exp_config[0][0]['metadata'] assert db_config['pool_size'] == exp_config[0][0]['pool_size'] assert db_config['max_trials'] == exp_config[0][0]['max_trials'] assert db_config['algorithms'] == exp_config[0][0]['algorithms']
def test_trials_interrupted_sigterm(config, monkeypatch): """Check if a trial is set as interrupted when a signal is raised.""" def mock_popen(*args, **kwargs): os.kill(os.getpid(), signal.SIGTERM) exp = ExperimentBuilder().build_from(config) monkeypatch.setattr(subprocess.Popen, "wait", mock_popen) trial = tuple_to_trial((1.0, ), exp.space) exp.register_trial(trial) con = Consumer(exp) with pytest.raises(KeyboardInterrupt): con.consume(trial) trials = exp.fetch_trials_by_status('interrupted') assert len(trials) assert trials[0].id == trial.id
def test_trials_interrupted_keyboard_int(config, monkeypatch): """Check if a trial is set as interrupted when a KeyboardInterrupt is raised.""" def mock_Popen(*args, **kwargs): raise KeyboardInterrupt exp = ExperimentBuilder().build_from(config) monkeypatch.setattr(consumer.subprocess, "Popen", mock_Popen) trial = tuple_to_trial((1.0, ), exp.space) exp.register_trial(trial) con = Consumer(exp) with pytest.raises(KeyboardInterrupt): con.consume(trial) trials = exp.fetch_trials_by_status('interrupted') assert len(trials) assert trials[0].id == trial.id
def test_fetch_local_config(config_file): """Test local config (default, env_vars, cmdconfig, cmdargs)""" cmdargs = {"config": config_file} local_config = ExperimentBuilder().fetch_full_config(cmdargs, use_db=False) assert local_config['algorithms'] == 'random' assert local_config['database']['host'] == 'mongodb://*****:*****@localhost' assert local_config['database']['name'] == 'orion_test' assert local_config['database']['type'] == 'mongodb' assert local_config['max_trials'] == 100 assert local_config['name'] == 'voila_voici' assert local_config['pool_size'] == 1
def test_fetch_full_config_no_hit(config_file, exp_config, random_dt): """Verify full config when experiment not in db""" cmdargs = {'name': 'supernaekei', 'config': config_file} full_config = ExperimentBuilder().fetch_full_config(cmdargs) assert full_config['name'] == 'supernaekei' assert full_config['algorithms'] == 'random' assert full_config['max_trials'] == 100 assert full_config['name'] == 'supernaekei' assert full_config['pool_size'] == 1 assert full_config['metadata']['user'] == 'tsirif' assert 'datetime' not in full_config['metadata'] assert 'refers' not in full_config
def test_build_view_from(config_file, create_db_instance, exp_config, random_dt): """Try building experiment view when in db""" cmdargs = {'name': 'supernaedo2', 'config': config_file} exp_view = ExperimentBuilder().build_view_from(cmdargs) assert exp_view._experiment._init_done is False assert get_view_db(exp_view) is create_db_instance assert exp_view._id == exp_config[0][0]['_id'] assert exp_view.name == exp_config[0][0]['name'] assert exp_view.configuration['refers'] == exp_config[0][0]['refers'] assert exp_view.metadata == exp_config[0][0]['metadata'] assert exp_view.pool_size == exp_config[0][0]['pool_size'] assert exp_view.max_trials == exp_config[0][0]['max_trials']