def build_view_from(self, cmdargs): """Build an experiment view based on full configuration. .. seealso:: `orion.core.io.experiment_builder` for more information on the hierarchy of configurations. :class:`orion.core.worker.experiment.ExperimentView` for more information on the experiment view object. """ local_config = self.fetch_full_config(cmdargs, use_db=False) db_opts = local_config['database'] dbtype = db_opts.pop('type') if local_config.get("debug"): dbtype = "EphemeralDB" # Information should be enough to infer experiment's name. log.debug("Creating %s database client with args: %s", dbtype, db_opts) try: Database(of_type=dbtype, **db_opts) except ValueError: if Database().__class__.__name__.lower() != dbtype.lower(): raise exp_name = local_config['name'] if exp_name is None: raise RuntimeError("Could not infer experiment's name. " "Please use either `name` cmd line arg or provide " "one in orion's configuration file.") return ExperimentView(local_config["name"], local_config.get('user', None))
def three_experiments_same_name_with_trials(two_experiments_same_name, storage): """Create three experiments with the same name but different versions.""" orion.core.cli.main([ "hunt", "--init-only", "-n", "test_single_exp", "./black_box.py", "--x~uniform(0,1)", "--y~normal(0,1)", "--z~+normal(0,1)", ]) ensure_deterministic_id("test_single_exp", storage, version=3) exp = experiment_builder.build(name="test_single_exp", version=1) exp2 = experiment_builder.build(name="test_single_exp", version=2) exp3 = experiment_builder.build(name="test_single_exp", version=3) x = {"name": "/x", "type": "real"} y = {"name": "/y", "type": "real"} z = {"name": "/z", "type": "real"} x_value = 0 for status in Trial.allowed_stati: x["value"] = x_value y["value"] = x_value * 10 z["value"] = x_value * 100 trial = Trial(experiment=exp.id, params=[x], status=status) trial2 = Trial(experiment=exp2.id, params=[x, y], status=status) trial3 = Trial(experiment=exp3.id, params=[x, y, z], status=status) Database().write("trials", trial.to_dict()) Database().write("trials", trial2.to_dict()) Database().write("trials", trial3.to_dict()) x_value += 1
def __init__(self, database=None, setup=True): if database is not None: setup_database(database) self._db = Database() if setup: self._setup_db()
def __init__(self, config=None, setup=True): if config is not None: setup_database(config) self._db = Database() if setup: self._setup_db()
def create_db_instance(null_db_instances, clean_db): """Create and save a singleton database instance.""" try: db = Database(of_type='MongoDB', name='orion_test', username='******', password='******') except ValueError: db = Database() return db
def __init__(self, name, user=None): """Initialize an Experiment object with primary key (:attr:`name`, :attr:`user`). Try to find an entry in `Database` with such a key and config this object from it import, if successful. Else, init with default/empty values and insert new entry with this object's attributes in database. .. note:: Practically initialization has not finished until `config`'s setter is called. :param name: Describe a configuration with a unique identifier per :attr:`user`. :type name: str """ log.debug("Creating Experiment object with name: %s", name) self._init_done = False self._db = Database() # fetch database instance self._setup_db() # build indexes for collections self._id = None self.name = name self._node = None self.refers = {} if user is None: user = getpass.getuser() self.metadata = {'user': user} self.pool_size = None self.max_trials = None self.algorithms = None self.working_dir = None self.producer = {'strategy': None} config = self._db.read('experiments', { 'name': name, 'metadata.user': user }) if config: log.debug( "Found existing experiment, %s, under user, %s, registered in database.", name, user) if len(config) > 1: log.warning( "Many (%s) experiments for (%s, %s) are available but " "only the most recent one can be accessed. " "Experiment branches will be supported soon.", len(config), name, user) config = sorted(config, key=lambda x: x['metadata']['datetime'], reverse=True)[0] for attrname in self.__slots__: if not attrname.startswith('_') and attrname in config: setattr(self, attrname, config[attrname]) self._id = config['_id'] self._last_fetched = self.metadata.get("datetime", datetime.datetime.utcnow())
def test_instantiation_and_singleton(self): """Test create just one object, that object persists between calls.""" database = Database(of_type="PickledDB", name="orion_test") assert isinstance(database, PickledDB) assert database is PickledDB() assert database is Database() with pytest.raises(SingletonAlreadyInstantiatedError): Database("fire", [], {"it_matters": "it's singleton"})
def db_instance(null_db_instances): """Create and save a singleton database instance.""" try: db = Database(of_type="MongoDB", name="orion_test", username="******", password="******") except ValueError: db = Database() return db
def test_instatiation_and_singleton(self): """Test create just one object, that object persists between calls.""" database = Database(of_type='MongoDB', name='orion_test', username='******', password='******') assert isinstance(database, MongoDB) assert database is MongoDB() assert database is Database() with pytest.raises(ValueError) as exc_info: Database('fire', [], {'it_matters': 'it\'s singleton'}) assert 'singleton' in str(exc_info.value)
def test_fix_lost_trials(self, hacked_exp, random_dt): """Test that a running trial with an old heartbeat is set to interrupted.""" exp_query = {'experiment': hacked_exp.id} trial = hacked_exp.fetch_trials(exp_query)[0] heartbeat = random_dt - datetime.timedelta(seconds=180) Database().write('trials', { 'status': 'reserved', 'heartbeat': heartbeat }, { 'experiment': hacked_exp.id, '_id': trial.id }) exp_query['status'] = 'reserved' exp_query['_id'] = trial.id assert len(hacked_exp.fetch_trials(exp_query)) == 1 hacked_exp.fix_lost_trials() assert len(hacked_exp.fetch_trials(exp_query)) == 0 exp_query['status'] = 'interrupted' assert len(hacked_exp.fetch_trials(exp_query)) == 1
def insert_trials(experiment_name, points, cmdconfig=None, raise_exc=True): """Insert sets of parameters manually, defined in `points`, as new trials for the experiment name, `experiment_name`. :param experiment_name: Name of the experiment which the new trials are going to be associated with :param points: list of tuples in agreement with experiment's parameter space :param raise_exc: whether an inappropriate tuple of parameters will raise an exception or it will be ignored .. note:: If `raise_exc` is True, no set of parameters will be inserted. If it is False, only the valid ones will be inserted; the rest will be ignored. .. note:: This cannot be used to prepopulate a future experiment. So, an experiment with `experiment_name` should already be configured in the database. """ cmdconfig = cmdconfig if cmdconfig else {} config = resolve_config.fetch_default_options( ) # Get database perhaps from default locs config = resolve_config.merge_env_vars( config) # Get database perhaps from env vars tmpconfig = resolve_config.merge_orion_config(config, dict(), cmdconfig, dict()) db_opts = tmpconfig['database'] db_type = db_opts.pop('type') try: Database(of_type=db_type, **db_opts) except SingletonError: pass experiment = Experiment(experiment_name) # Configuration is completely taken from the database if experiment.status is None: raise ValueError( "No experiment named '{}' could be found.".format(experiment_name)) experiment.configure(experiment.configuration) valid_points = [] print(experiment.space) for point in points: try: assert point in experiment.space valid_points.append(point) except AssertionError: if raise_exc: raise if not valid_points: return new_trials = list( map(lambda data: format_trials.tuple_to_trial(data, experiment.space), valid_points)) experiment.register_trials(new_trials)
def _infer_experiment(cmd_args, file_config): # Initialize configuration dictionary. # Fetch info from defaults and configurations from default locations. expconfig = resolve_config.fetch_default_options() # Fetch orion system variables (database and resource information) # See :const:`orion.core.io.resolve_config.ENV_VARS` for environmental # variables used expconfig = resolve_config.merge_env_vars(expconfig) # Initialize singleton database object tmpconfig = resolve_config.merge_orion_config(expconfig, dict(), file_config, cmd_args) db_opts = tmpconfig['database'] dbtype = db_opts.pop('type') log.debug("Creating %s database client with args: %s", dbtype, db_opts) Database(of_type=dbtype, **db_opts) # Information should be enough to infer experiment's name. exp_name = tmpconfig['name'] if exp_name is None: raise RuntimeError("Could not infer experiment's name. " "Please use either `name` cmd line arg or provide " "one in orion's configuration file.") experiment = create_experiment(exp_name, expconfig, file_config, cmd_args) return experiment
def main(args): """List all experiments inside database.""" builder = ExperimentBuilder() config = builder.fetch_full_config(args, use_db=False) builder.setup_database(config) query = {} if args['name']: query['name'] = args['name'] experiments = Database().read("experiments", query) if args['name']: root_experiments = experiments else: root_experiments = [ exp for exp in experiments if exp['refers'].get('root_id', exp['_id']) == exp['_id'] ] for root_experiment in root_experiments: root = EVCBuilder().build_view_from({ 'name': root_experiment['name'] }).node print_tree(root)
def consume(self, trial): """Execute user's script as a block box using the options contained within `trial`. :type trial: `orion.core.worker.trial.Trial` """ log.debug("### Create new directory at '%s':", self.working_dir) temp_dir = self.experiment.working_dir is None prefix = self.experiment.name + "_" suffix = trial.id with WorkingDir(self.working_dir, temp_dir, prefix=prefix, suffix=suffix) as workdirname: log.debug("## New consumer context: %s", workdirname) completed_trial = self._consume(trial, workdirname) if completed_trial is not None: log.debug("### Register successfully evaluated %s.", completed_trial) self.experiment.push_completed_trial(completed_trial) else: log.debug("### Save %s as broken.", trial) trial.status = 'broken' Database().write('trials', trial.to_dict(), query={'_id': trial.id})
def infer_experiment(): """Use `orion.core.resolve_config` to organize how configuration is built.""" # Fetch experiment name, user's script path and command line arguments # Use `-h` option to show help cmdargs, cmdconfig = resolve_config.fetch_orion_args(CLI_DOC_HEADER) # Initialize configuration dictionary. # Fetch info from defaults and configurations from default locations. expconfig = resolve_config.fetch_default_options() # Fetch orion system variables (database and resource information) # See :const:`orion.core.io.resolve_config.ENV_VARS` for environmental # variables used expconfig = resolve_config.merge_env_vars(expconfig) # Initialize singleton database object tmpconfig = resolve_config.merge_orion_config(expconfig, dict(), cmdconfig, cmdargs) db_opts = tmpconfig['database'] dbtype = db_opts.pop('type') log.debug("Creating %s database client with args: %s", dbtype, db_opts) Database(of_type=dbtype, **db_opts) # Information should be enough to infer experiment's name. exp_name = tmpconfig['name'] if exp_name is None: raise RuntimeError("Could not infer experiment's name. " "Please use either `name` cmd line arg or provide " "one in orion's configuration file.") experiment = create_experiment(exp_name, expconfig, cmdconfig, cmdargs) return experiment
def test_setup_database_custom(): """Test setup with local configuration""" update_singletons() setup_database({"type": "pickleddb", "host": "test.pkl"}) database = Database() assert isinstance(database, PickledDB) assert database.host == "test.pkl"
def test_workon(database): """Test scenario having a configured experiment already setup.""" try: Database(of_type='MongoDB', name='orion_test', username='******', password='******') except (TypeError, ValueError): pass experiment = Experiment('voila_voici') config = experiment.configuration config['algorithms'] = {'gradient_descent': {'learning_rate': 0.1}} config['pool_size'] = 1 config['max_trials'] = 100 config['metadata']['user_script'] = os.path.abspath( os.path.join(os.path.dirname(__file__), "black_box.py")) config['metadata']['user_args'] = ["-x~uniform(-50, 50)"] experiment.configure(config) workon(experiment) exp = list(database.experiments.find({'name': 'voila_voici'})) assert len(exp) == 1 exp = exp[0] assert '_id' in exp exp_id = exp['_id'] assert exp['name'] == 'voila_voici' assert exp['pool_size'] == 1 assert exp['max_trials'] == 100 assert exp['algorithms'] == { 'gradient_descent': { 'learning_rate': 0.1, 'dx_tolerance': 1e-7 } } assert 'user' in exp['metadata'] assert 'datetime' in exp['metadata'] assert 'user_script' in exp['metadata'] assert os.path.isabs(exp['metadata']['user_script']) assert exp['metadata']['user_args'] == ['-x~uniform(-50, 50)'] trials = list(database.trials.find({'experiment': exp_id})) assert len(trials) <= 15 trials = list(sorted(trials, key=lambda trial: trial['submit_time'])) assert trials[-1]['status'] == 'completed' for result in trials[-1]['results']: assert result['type'] != 'constraint' if result['type'] == 'objective': assert abs(result['value'] - 23.4) < 1e-6 assert result['name'] == 'example_objective' elif result['type'] == 'gradient': res = numpy.asarray(result['value']) assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7 assert result['name'] == 'example_gradient' params = trials[-1]['params'] assert len(params) == 1 assert params[0]['name'] == '/x' assert params[0]['type'] == 'real' assert (params[0]['value'] - 34.56789) < 1e-5
def single_with_trials(single_without_success): """Create an experiment with all types of trials.""" exp = ExperimentBuilder().build_from({'name': 'test_single_exp'}) x = {'name': '/x', 'type': 'real', 'value': 100} results = {"name": "obj", "type": "objective", "value": 0} trial = Trial(experiment=exp.id, params=[x], status='completed', results=[results]) Database().write('trials', trial.to_dict())
def family_with_trials(two_experiments): """Create two related experiments with all types of trials.""" exp = experiment_builder.build(name="test_double_exp") exp2 = experiment_builder.build(name="test_double_exp_child") x = {"name": "/x", "type": "real"} y = {"name": "/y", "type": "real"} x_value = 0 for status in Trial.allowed_stati: x["value"] = x_value y["value"] = x_value trial = Trial(experiment=exp.id, params=[x], status=status) x["value"] = x_value trial2 = Trial(experiment=exp2.id, params=[x, y], status=status) x_value += 1 Database().write("trials", trial.to_dict()) Database().write("trials", trial2.to_dict())
def family_with_trials(two_experiments): """Create two related experiments with all types of trials.""" exp = experiment_builder.build(name='test_double_exp') exp2 = experiment_builder.build(name='test_double_exp_child') x = {'name': '/x', 'type': 'real'} y = {'name': '/y', 'type': 'real'} x_value = 0 for status in Trial.allowed_stati: x['value'] = x_value y['value'] = x_value trial = Trial(experiment=exp.id, params=[x], status=status) x['value'] = x_value trial2 = Trial(experiment=exp2.id, params=[x, y], status=status) x_value += 1 Database().write('trials', trial.to_dict()) Database().write('trials', trial2.to_dict())
def test_setup_database_bad_config_override(): """Test setup with different config than existing singleton""" update_singletons() setup_database({"type": "pickleddb", "host": "test.pkl"}) database = Database() assert isinstance(database, PickledDB) with pytest.raises(SingletonAlreadyInstantiatedError): setup_database({"type": "pickleddb", "host": "other.pkl"})
def with_experiment_missing_conf_file(monkeypatch, one_experiment): """Create an experiment without trials.""" exp = experiment_builder.build(name="test_single_exp", version=1) conf_file = "idontexist.yaml" exp.metadata["user_config"] = conf_file exp.metadata["user_args"] += ["--config", conf_file] Database().write("experiments", exp.configuration, query={"_id": exp.id}) return exp
def test_setup_database_bad_override(): """Test setup with different type than existing singleton""" update_singletons() setup_database({"type": "pickleddb", "host": "test.pkl"}) database = Database() assert isinstance(database, PickledDB) with pytest.raises(SingletonAlreadyInstantiatedError) as exc: setup_database({"type": "mongodb"}) assert exc.match("A singleton instance of \(type: Database\)")
def test_empty_first_call(self): """Should not be able to make first call without any arguments. Hegelian Ontology Primer ------------------------ Type indeterminate <-> type abstracted from its property <-> No type """ with pytest.raises(SingletonNotInstantiatedError): Database()
def test_empty_first_call(self): """Should not be able to make first call without any arguments. Hegelian Ontology Primer ------------------------ Type indeterminate <-> type abstracted from its property <-> No type """ with pytest.raises(TypeError) as exc_info: Database() assert 'positional argument' in str(exc_info.value)
def single_with_trials(single_without_success): """Create an experiment with all types of trials.""" exp = experiment_builder.build(name="test_single_exp") x = {"name": "/x", "type": "real", "value": 100} results = {"name": "obj", "type": "objective", "value": 0} trial = Trial(experiment=exp.id, params=[x], status="completed", results=[results]) Database().write("trials", trial.to_dict())
def trial(exp): """Return a Trial which is registered in DB.""" trial = tuple_to_trial((1.0, ), exp.space) heartbeat = datetime.datetime.utcnow() trial.experiment = exp.id trial.status = 'reserved' trial.heartbeat = heartbeat Database().write('trials', trial.to_dict()) return trial
def setup_database(self, config): """Create the Database instance from a configuration. Parameters ---------- config: dict Configuration for the database. """ db_opts = config['database'] dbtype = db_opts.pop('type') if config.get("debug"): dbtype = "EphemeralDB" log.debug("Creating %s database client with args: %s", dbtype, db_opts) try: Database(of_type=dbtype, **db_opts) except ValueError: if Database().__class__.__name__.lower() != dbtype.lower(): raise
def test_reserve_none(self): """Find nothing, return None.""" try: Database(of_type='MongoDB', name='orion_test', username='******', password='******') except (TypeError, ValueError): pass exp = Experiment('supernaekei') trial = exp.reserve_trial() assert trial is None
def test_broken_property(hacked_exp): """Check experiment stopping conditions for maximum number of broken.""" assert not hacked_exp.is_broken trials = hacked_exp.fetch_trials({})[:3] query = {'experiment': hacked_exp.id} for trial in trials: query['_id'] = trial.id Database().write('trials', {'status': 'broken'}, query) assert hacked_exp.is_broken