def test_code_change_break_backward(create_db_instance): """Test that no trials pass to parent when code change type is 'break'""" experiment_name = 'supernaedo2.3.1' root_name = 'supernaedo2.3.1' leaf_names = ['supernaedo2.3.1.3'] experiment = ExperimentView(experiment_name) exp_node = build_trimmed_tree(experiment, root_name, leaf_names) assert exp_node.item.name == experiment_name assert exp_node.children[0].item.name == leaf_names[0] # 2.3 # | # 2.3.1 assert len(list(exp_node.root)) == 2 experiment.connect_to_version_control_tree(exp_node) query = {'status': 'completed'} children_trials = exp_node.children[0].item.fetch_trials(query) assert len(children_trials) == 1 assert len(exp_node.item.fetch_trials(query)) == 2 adapter = exp_node.children[0].item.refers['adapter'] assert adapter.adapters[0].change_type == CodeChange.BREAK adapted_children_trials = adapter.backward(children_trials) assert len(adapted_children_trials) == 0 assert len(experiment.fetch_trials_tree(query)) == 2 + 0
def test_experiment_view_not_modified(self, exp_config, monkeypatch): """Experiment should not be modified if fetched in another verion of Oríon. When loading a view the original config is used to configure the experiment, but this process may modify the config if the version of Oríon is different. This should not be saved in database. """ terrible_message = 'oh no, I have been modified!' original_configuration = ExperimentView('supernaedo2').configuration def modified_configuration(self): mocked_config = copy.deepcopy(original_configuration) mocked_config['metadata']['datetime'] = terrible_message return mocked_config with monkeypatch.context() as m: m.setattr(Experiment, 'configuration', property(modified_configuration)) exp = ExperimentView('supernaedo2') # The mock is still in place and overwrites the configuration assert exp.configuration['metadata'][ 'datetime'] == terrible_message # The mock is reverted and original config is returned, but modification is still in # metadata assert exp.metadata['datetime'] == terrible_message # Loading again from DB confirms the DB was not overwritten reloaded_exp = ExperimentView('supernaedo2') assert reloaded_exp.configuration['metadata'][ 'datetime'] != terrible_message
def test_full_forward_full_backward(create_db_instance): """Test that trials are adapted properly forward from parent and backward from leafs""" experiment_name = 'supernaedo2.3' root_name = 'supernaedo2' leaf_names = [] experiment = ExperimentView(experiment_name) exp_node = build_trimmed_tree(experiment, root_name, leaf_names) assert exp_node.item.name == experiment_name assert exp_node.parent.name == root_name assert exp_node.children[0].item.name == 'supernaedo2.3.1' assert exp_node.children[0].children[0].item.name == 'supernaedo2.3.1.1' assert exp_node.children[0].children[1].item.name == 'supernaedo2.3.1.2' assert exp_node.children[0].children[2].item.name == 'supernaedo2.3.1.3' # 2 # | # 2.3 # | # 2.3.1 # | \ \ # 2.3.1.1 2.3.1.2 2.3.1.3 assert len(list(exp_node.root)) == 6 experiment.connect_to_version_control_tree(exp_node) query = {'status': 'completed'} assert len(exp_node.parent.item.fetch_trials(query)) == 6 assert len(exp_node.item.fetch_trials(query)) == 4 assert len(exp_node.children[0].item.fetch_trials(query)) == 2 assert len(exp_node.children[0].children[0].item.fetch_trials(query)) == 1 assert len(exp_node.children[0].children[1].item.fetch_trials(query)) == 1 assert len(exp_node.children[0].children[2].item.fetch_trials(query)) == 1 assert len(experiment.fetch_trials_tree(query)) == 6 + 4 + 1 + 1 + 0 + 0
def test_parent_parent_fetch_trials(create_db_instance): """Test that experiment fetch trials from grand parent properly (adapters are muted)""" experiment_name = 'supernaedo2.3.1' root_name = 'supernaedo2' leaf_names = ['supernaedo2.3.1'] experiment = ExperimentView(experiment_name) exp_node = build_trimmed_tree(experiment, root_name, leaf_names) assert exp_node.item.name == experiment_name assert exp_node.parent.parent.item.name == root_name assert len(exp_node.children) == 0 # 2 # | # 2.3 # | # 2.3.1 assert len(list(exp_node.root)) == 3 experiment.connect_to_version_control_tree(exp_node) for node in exp_node.root: node.item._experiment.refers['adapter'] = Adapter.build([]) query = {'status': 'completed'} assert len(exp_node.parent.parent.item.fetch_trials(query)) == 6 assert len(exp_node.parent.item.fetch_trials(query)) == 4 assert len(exp_node.item.fetch_trials(query)) == 2 assert len(experiment.fetch_trials_tree(query)) == 6 + 4 + 2
def test_existing_experiment_view(self, create_db_instance, exp_config): """Hit exp_name + user's name in the db, fetch most recent entry.""" exp = ExperimentView('supernaedo2') assert exp._experiment._init_done is False assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] # TODO: Views are not fully configured until configuration is refactored # assert exp.algorithms.configuration == exp_config[0][0]['algorithms'] with pytest.raises(AttributeError): exp.this_is_not_in_config = 5 # Test that experiment.update_completed_trial indeed exists exp._experiment.update_completed_trial with pytest.raises(AttributeError): exp.update_completed_trial with pytest.raises(AttributeError): exp.register_trial with pytest.raises(AttributeError): exp.reserve_trial
def test_full_forward(create_db_instance): """Test that trials are adapted properly down to leaf""" experiment_name = 'supernaedo2.3.1.1' root_name = None leaf_names = [] experiment = ExperimentView(experiment_name) exp_node = build_trimmed_tree(experiment, root_name, leaf_names) assert exp_node.item.name == experiment_name assert exp_node.parent.item.name == 'supernaedo2.3.1' assert exp_node.parent.parent.item.name == 'supernaedo2.3' assert exp_node.parent.parent.parent.item.name == 'supernaedo2' # 2 # | # 2.3 # | # 2.3.1 # | # 2.3.1.1 assert len(list(exp_node.root)) == 4 experiment.connect_to_version_control_tree(exp_node) query = {'status': 'completed'} assert len(exp_node.item.fetch_trials(query)) == 1 assert len(exp_node.parent.item.fetch_trials(query)) == 2 assert len(exp_node.parent.parent.item.fetch_trials(query)) == 4 assert len(exp_node.parent.parent.parent.item.fetch_trials(query)) == 6 assert len(experiment.fetch_trials_tree(query)) == 2 + 1 + 2 + 1
def test_code_change_unsure_forward(create_db_instance): """Test that all trials pass to children when code change type is 'unsure'""" experiment_name = 'supernaedo2.3.1.2' root_name = 'supernaedo2.3.1' leaf_names = ['supernaedo2.3.1.2'] experiment = ExperimentView(experiment_name) exp_node = build_trimmed_tree(experiment, root_name, leaf_names) assert exp_node.item.name == experiment_name assert exp_node.parent.item.name == root_name # 2.3.1 # | # 2.3.1.1 assert len(list(exp_node.root)) == 2 experiment.connect_to_version_control_tree(exp_node) query = {'status': 'completed'} parent_trials = exp_node.parent.item.fetch_trials(query) assert len(parent_trials) == 2 assert len(exp_node.item.fetch_trials(query)) == 1 adapter = experiment.refers['adapter'] assert adapter.adapters[0].change_type == CodeChange.UNSURE adapted_parent_trials = adapter.forward(parent_trials) assert len(adapted_parent_trials) == 2 assert len(experiment.fetch_trials_tree(query)) == 2 + 1
def test_existing_experiment_view(self, create_db_instance, exp_config): """Hit exp_name + user's name in the db, fetch most recent entry.""" exp = ExperimentView('supernaedo2') assert exp._experiment._init_done is True assert exp._experiment._db._database is create_db_instance assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.configuration['refers'] == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] assert exp._experiment._last_fetched == exp_config[0][0]['metadata'][ 'datetime'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] assert exp.algorithms.configuration == exp_config[0][0]['algorithms'] with pytest.raises(AttributeError): exp.this_is_not_in_config = 5 # Test that experiment.push_completed_trial indeed exists exp._experiment.push_completed_trial with pytest.raises(AttributeError): exp.push_completed_trial with pytest.raises(AttributeError): exp.register_trial with pytest.raises(AttributeError): exp.reserve_trial
def test_prior_change_forward(create_db_instance): """Test that trials from parent only pass to children if valid in the new prior""" experiment_name = 'supernaedo2.3.1' root_name = 'supernaedo2.3' leaf_names = ['supernaedo2.3.1'] experiment = ExperimentView(experiment_name) exp_node = build_trimmed_tree(experiment, root_name, leaf_names) assert exp_node.item.name == experiment_name assert exp_node.parent.item.name == root_name # 2.3 # | # 2.3.1 assert len(list(exp_node.root)) == 2 experiment.connect_to_version_control_tree(exp_node) query = {'status': 'completed'} parent_trials = exp_node.parent.item.fetch_trials(query) assert len(parent_trials) == 4 assert len(exp_node.item.fetch_trials(query)) == 2 adapter = experiment.refers['adapter'] adapted_parent_trials = adapter.forward(parent_trials) assert len(adapted_parent_trials) == 1 assert len(experiment.fetch_trials_tree(query)) == 2 + 1
def test_prior_change_backward(create_db_instance): """Test that all encoding are renamed to encoding_layer in parent""" experiment_name = 'supernaedo2.3' root_name = 'supernaedo2.3' leaf_names = ['supernaedo2.3.1'] experiment = ExperimentView(experiment_name) exp_node = build_trimmed_tree(experiment, root_name, leaf_names) assert exp_node.item.name == experiment_name assert exp_node.children[0].item.name == leaf_names[0] # 2.3 # | # 2.3.1 assert len(list(exp_node.root)) == 2 experiment.connect_to_version_control_tree(exp_node) query = {'status': 'completed'} children_trials = exp_node.children[0].item.fetch_trials(query) assert len(children_trials) == 2 assert len(exp_node.item.fetch_trials(query)) == 4 adapter = exp_node.children[0].item.refers['adapter'] adapted_children_trials = adapter.backward(children_trials) assert len(adapted_children_trials) == 1 assert len(experiment.fetch_trials_tree(query)) == 4 + 1
def test_renaming_forward(create_db_instance): """Test that all encoding_layer are renamed to encoding in children""" experiment_name = 'supernaedo2.3' root_name = 'supernaedo2' leaf_names = ['supernaedo2.3'] experiment = ExperimentView(experiment_name) exp_node = build_trimmed_tree(experiment, root_name, leaf_names) assert exp_node.item.name == experiment_name assert exp_node.parent.item.name == root_name # 2 # | # 2.1 assert len(list(exp_node.root)) == 2 experiment.connect_to_version_control_tree(exp_node) query = {'status': 'completed'} parent_trials = exp_node.parent.item.fetch_trials(query) assert len(parent_trials) == 6 assert len(exp_node.item.fetch_trials(query)) == 4 assert all((trial._params[0].name == "/encoding_layer") for trial in parent_trials) adapter = experiment.refers['adapter'] adapted_parent_trials = adapter.forward(parent_trials) assert len(adapted_parent_trials) == 6 assert all((trial._params[0].name == "/encoding") for trial in adapted_parent_trials) assert len(experiment.fetch_trials_tree(query)) == 6 + 4
def test_deletion_adapter_backward(create_db_instance): """Test that all decoding_layer are passed with gru to parent""" experiment_name = 'supernaedo2' root_name = 'supernaedo2' leaf_names = ['supernaedo2.1'] experiment = ExperimentView(experiment_name) exp_node = build_trimmed_tree(experiment, root_name, leaf_names) assert exp_node.item.name == experiment_name assert exp_node.children[0].item.name == leaf_names[0] # 2 # | # 2.1 assert len(list(exp_node.root)) == 2 experiment.connect_to_version_control_tree(exp_node) query = {'status': 'completed'} assert len(exp_node.item.fetch_trials(query)) == 6 assert len(exp_node.children[0].item.fetch_trials(query)) == 1 adapter = exp_node.children[0].item.refers['adapter'] assert len(adapter.backward(exp_node.children[0].item.fetch_trials(query))) == 1 assert len(experiment.fetch_trials_tree(query)) == 6 + 1
def test_algo_change_backward(create_db_instance): """Test that all trials pass to parent when algorithm is changed""" experiment_name = 'supernaedo2.2' root_name = 'supernaedo2.2' leaf_names = ['supernaedo2.2.1'] experiment = ExperimentView(experiment_name) exp_node = build_trimmed_tree(experiment, root_name, leaf_names) assert exp_node.item.name == experiment_name assert exp_node.children[0].item.name == leaf_names[0] # 2.2 # | # 2.2.1 assert len(list(exp_node.root)) == 2 experiment.connect_to_version_control_tree(exp_node) query = {'status': 'completed'} children_trials = exp_node.children[0].item.fetch_trials(query) assert len(children_trials) == 1 assert len(exp_node.item.fetch_trials(query)) == 1 adapter = exp_node.children[0].item.refers['adapter'] adapted_children_trials = adapter.backward(children_trials) assert len(adapted_children_trials) == 1 assert len(experiment.fetch_trials_tree(query)) == 1 + 1
def test_fetch_completed_trials_from_view(hacked_exp, exp_config, random_dt): """Fetch a list of the unseen yet completed trials.""" experiment_view = ExperimentView(hacked_exp.name) experiment_view._experiment = hacked_exp trials = experiment_view.fetch_completed_trials() assert len(trials) == 3 assert trials[0].to_dict() == exp_config[1][0] assert trials[1].to_dict() == exp_config[1][2] assert trials[2].to_dict() == exp_config[1][1]
def item(self): """Get the experiment associated to the node Note that accessing `item` may trigger the lazy initialization of the experiment if it was not done already. """ if self._item is None: self._item = ExperimentView(self.name, version=self.version) self._item.connect_to_version_control_tree(self) return self._item
def test_fetch_completed_trials_from_view(): """Fetch a list of the unseen yet completed trials.""" non_completed_stati = ['new', 'interrupted', 'suspended', 'reserved'] stati = non_completed_stati + ['completed'] with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp_view = ExperimentView(exp) trials = exp_view.fetch_trials_by_status('completed') assert len(trials) == 1 assert trials[0].status == 'completed'
def test_experiment_view_stats(hacked_exp, exp_config, random_dt): """Check that property stats from view is consistent.""" experiment_view = ExperimentView(hacked_exp.name) experiment_view._experiment = hacked_exp stats = experiment_view.stats assert stats['trials_completed'] == 3 assert stats['best_trials_id'] == exp_config[1][1]['_id'] assert stats['best_evaluation'] == 2 assert stats['start_time'] == exp_config[0][3]['metadata']['datetime'] assert stats['finish_time'] == exp_config[1][2]['end_time'] assert stats['duration'] == stats['finish_time'] - stats['start_time'] assert len(stats) == 6
def test_view_algo_is_done_property(hacked_exp): """Check experiment's algo stopping conditions accessed from view.""" experiment_view = ExperimentView(hacked_exp.name) experiment_view._experiment = hacked_exp # Fully configure wrapper experiment (should normally occur inside ExperimentView.__init__ # but hacked_exp has been _hacked_ inside afterwards. hacked_exp.configure(hacked_exp.configuration) assert experiment_view.is_done is False hacked_exp.algorithms.algorithm.done = True assert experiment_view.is_done is True
def test_empty_experiment_view_due_to_username(self): """Hit exp_name, but user's name does not hit the db, create new entry.""" with pytest.raises(ValueError) as exc_info: ExperimentView('supernaedo2') assert ( "No experiment with given name 'supernaedo2' for user 'bouthilx'" in str(exc_info.value))
def build_view(name, version=None): """Build experiment view An experiment view provides all reading operations of standard experiment but prevents the modification of the experiment and its trials. Parameters ---------- name: str Name of the experiment to build version: int, optional Version to select. If None, last version will be selected. If version given is larger than largest version available, the largest version will be selected. """ db_config = fetch_config_from_db(name, version) if not db_config: message = ( "No experiment with given name '%s' and version '%s' inside database, " "no view can be created." % (name, version if version else '*')) raise ValueError(message) db_config.setdefault('version', 1) experiment = create_experiment(**db_config) return ExperimentView(experiment)
def build_view_from(self, cmdargs): """Build an experiment view based on full configuration. .. seealso:: `orion.core.io.experiment_builder` for more information on the hierarchy of configurations. :class:`orion.core.worker.experiment.ExperimentView` for more information on the experiment view object. """ local_config = self.fetch_full_config(cmdargs, use_db=False) self.setup_database(local_config) # Information should be enough to infer experiment's name. exp_name = local_config['name'] if exp_name is None: raise RuntimeError( "Could not infer experiment's name. " "Please use either `name` cmd line arg or provide " "one in orion's configuration file.") return ExperimentView(local_config["name"], local_config.get('user', None))
def build_view_from(self, cmdargs): """Build an experiment view based on full configuration. .. seealso:: `orion.core.io.experiment_builder` for more information on the hierarchy of configurations. :class:`orion.core.worker.experiment.ExperimentView` for more information on the experiment view object. """ local_config = self.fetch_full_config(cmdargs, use_db=False) db_opts = local_config['database'] dbtype = db_opts.pop('type') if local_config.get("debug"): dbtype = "EphemeralDB" # Information should be enough to infer experiment's name. log.debug("Creating %s database client with args: %s", dbtype, db_opts) try: Database(of_type=dbtype, **db_opts) except ValueError: if Database().__class__.__name__.lower() != dbtype.lower(): raise exp_name = local_config['name'] if exp_name is None: raise RuntimeError("Could not infer experiment's name. " "Please use either `name` cmd line arg or provide " "one in orion's configuration file.") return ExperimentView(local_config["name"], local_config.get('user', None))
def test_experiment_view_protocol_read_only(): """Verify that wrapper experiments' database is read-only""" exp = ExperimentView('supernaedo2') # Test that _protocol.update_trials indeed exists exp._experiment._storage._storage.update_trial with pytest.raises(AttributeError): exp._experiment._storage.update_trial
def test_experiment_view_db_read_only(): """Verify that wrapper experiments' database is read-only""" exp = ExperimentView('supernaedo2') # Test that database.write indeed exists exp._experiment._db._database.write with pytest.raises(AttributeError): exp._experiment._db.write
def test_view_is_done_property(hacked_exp): """Check experiment stopping conditions accessed from view.""" experiment_view = ExperimentView(hacked_exp.name) experiment_view._experiment = hacked_exp # Fully configure wrapper experiment (should normally occur inside ExperimentView.__init__ # but hacked_exp has been _hacked_ inside afterwards. hacked_exp.configure(hacked_exp.configuration) assert experiment_view.is_done is False with pytest.raises(AttributeError): experiment_view.max_trials = 2 hacked_exp.max_trials = 2 assert experiment_view.is_done is True
def test_experiment_view_protocol_read_only(): """Verify that wrapper experiments' protocol is read-only""" with OrionState(): exp = Experiment('supernaekei') exp_view = ExperimentView(exp) # Test that _protocol.set_trial_status indeed exists exp_view._experiment._storage._storage.set_trial_status with pytest.raises(AttributeError): exp_view._experiment._storage.set_trial_status
def test_experiment_view_stats(): """Check that property stats from view is consistent.""" NUM_COMPLETED = 3 stati = (['completed'] * NUM_COMPLETED) + (['reserved'] * 2) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.metadata = {'datetime': datetime.datetime.utcnow()} exp_view = ExperimentView(exp) stats = exp_view.stats assert stats['trials_completed'] == NUM_COMPLETED assert stats['best_trials_id'] == cfg.trials[3]['_id'] assert stats['best_evaluation'] == 0 assert stats['start_time'] == exp_view.metadata['datetime'] assert stats['finish_time'] == cfg.trials[0]['end_time'] assert stats['duration'] == stats['finish_time'] - stats['start_time'] assert len(stats) == 6
def test_view_is_done_property_no_pending(algorithm): """Check experiment stopping conditions from view when there is no pending trials.""" completed = ['completed'] * 10 broken = ['broken'] * 5 with OrionState(trials=generate_trials(completed + broken)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.algorithms = algorithm exp.max_trials = 100 exp_view = ExperimentView(exp) exp.algorithms = algorithm exp.max_trials = 15 # There is only 10 completed trials and algo not done. assert not exp_view.is_done exp.algorithms.algorithm.done = True # Algorithm is done and no pending trials assert exp_view.is_done
def test_view_is_done_property_with_pending(algorithm): """Check experiment stopping conditions from view when there is pending trials.""" completed = ['completed'] * 10 reserved = ['reserved'] * 5 with OrionState(trials=generate_trials(completed + reserved)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.algorithms = algorithm exp.max_trials = 10 exp_view = ExperimentView(exp) assert exp_view.is_done exp.max_trials = 15 # There is only 10 completed trials assert not exp_view.is_done exp.algorithms.algorithm.done = True # Algorithm is done but 5 trials are pending assert not exp_view.is_done
def test_init(init_full_x, create_db_instance): """Test if original experiment contains trial 0""" experiment = ExperimentView('full_x') pairs = get_name_value_pairs(experiment.fetch_trials({})) assert pairs == ((('/x', 0), ), )