def test_fetch_config_deprecate_tricky_names(monkeypatch, config_file): """Verify fetch_config assigns values properly for the similar names.""" def mocked_config(file_object): return { 'experiment': { 'worker_trials': 'should_be_ignored' }, 'max_trials': 'exp_max_trials', 'max_broken': 'exp_max_broken', 'worker_trials': 'worker_max_trials', 'name': 'exp_name' } monkeypatch.setattr('yaml.safe_load', mocked_config) config = resolve_config.fetch_config({"config": config_file}) assert config == { 'experiment': { 'name': 'exp_name', 'max_trials': 'exp_max_trials', 'max_broken': 'exp_max_broken' }, 'worker': { 'max_trials': 'worker_max_trials' } }
def test_fetch_config_deprecate_tricky_names(monkeypatch, config_file): """Verify fetch_config assigns values properly for the similar names.""" def mocked_config(file_object): return { "experiment": { "worker_trials": "should_be_ignored" }, "max_trials": "exp_max_trials", "max_broken": "exp_max_broken", "worker_trials": "worker_max_trials", "name": "exp_name", } monkeypatch.setattr("yaml.safe_load", mocked_config) config = resolve_config.fetch_config({"config": config_file}) assert config == { "experiment": { "name": "exp_name", "max_trials": "exp_max_trials", "max_broken": "exp_max_broken", }, "worker": { "max_trials": "worker_max_trials" }, }
def test_fetch_config_deprecated_max_trials(monkeypatch, config_file): """Verify fetch_config will overwrite deprecated value if also properly defined.""" def mocked_config(file_object): return {"experiment": {"max_trials": 10}, "max_trials": 20} monkeypatch.setattr("yaml.safe_load", mocked_config) config = resolve_config.fetch_config({"config": config_file}) assert config == {"experiment": {"max_trials": 10}}
def test_fetch_config(config_file): """Verify fetch_config returns valid dictionnary""" config = resolve_config.fetch_config({"config": config_file}) assert config['algorithms'] == 'random' assert config['database']['host'] == 'mongodb://*****:*****@localhost' assert config['database']['name'] == 'orion_test' assert config['database']['type'] == 'mongodb' assert config['max_trials'] == 100 assert config['name'] == 'voila_voici' assert config['pool_size'] == 1
def test_fetch_config_underscore(monkeypatch, config_file): """Verify fetch_config supports underscore as well.""" def mocked_config(file_object): return {"experiment": {"max_broken": 10, "algorithms": {"dont-change": "me"}}} monkeypatch.setattr("yaml.safe_load", mocked_config) config = resolve_config.fetch_config({"config": config_file}) assert config == { "experiment": {"max_broken": 10, "algorithms": {"dont-change": "me"}} }
def get_cmd_config(cmdargs): """Fetch configuration defined by commandline and local configuration file. Arguments of commandline have priority over options in configuration file. """ cmdargs = resolve_config.fetch_config_from_cmdargs(cmdargs) cmd_config = resolve_config.fetch_config(cmdargs) cmd_config = resolve_config.merge_configs(cmd_config, cmdargs) cmd_config.update(cmd_config.pop("experiment", {})) cmd_config["user_script_config"] = cmd_config.get("worker", {}).get( "user_script_config", None ) cmd_config["branching"] = cmd_config.pop("evc", {}) # TODO: We should move branching specific stuff below in a centralized place for EVC stuff. if ( cmd_config["branching"].get("auto_resolution", False) and cmdargs.get("manual_resolution", None) is None ): cmd_config["branching"]["manual_resolution"] = False non_monitored_arguments = cmdargs.get("non_monitored_arguments") if non_monitored_arguments: cmd_config["branching"][ "non_monitored_arguments" ] = non_monitored_arguments.split(":") # TODO: user_args won't be defined if reading from DB only (`orion hunt -n <exp> ` alone) metadata = resolve_config.fetch_metadata( cmd_config.get("user"), cmd_config.get("user_args"), cmd_config.get("user_script_config"), ) cmd_config["metadata"] = metadata cmd_config.pop("config", None) cmd_config["space"] = cmd_config["metadata"].get("priors", None) backward.update_db_config(cmd_config) return cmd_config
def test_fetch_config(config_file): """Verify fetch_config returns valid dictionnary""" config = resolve_config.fetch_config({"config": config_file}) assert config.pop("storage") == { "database": { "host": "mongodb://*****:*****@localhost", "name": "orion_test", "type": "mongodb", } } assert config.pop("experiment") == { "max_trials": 100, "max_broken": 5, "name": "voila_voici", "algorithms": "random", } assert config == {}
def get_cmd_config(cmdargs): """Fetch configuration defined by commandline and local configuration file. Arguments of commandline have priority over options in configuration file. """ cmdargs = resolve_config.fetch_config_from_cmdargs(cmdargs) cmd_config = resolve_config.fetch_config(cmdargs) cmd_config = resolve_config.merge_configs(cmd_config, cmdargs) cmd_config.update(cmd_config.pop('experiment', {})) cmd_config['branching'] = cmd_config.pop('evc', {}) metadata = resolve_config.fetch_metadata(cmd_config.get('user'), cmd_config.get('user_args')) cmd_config['metadata'] = metadata cmd_config.pop('config', None) backward.populate_space(cmd_config) backward.update_db_config(cmd_config) return cmd_config
def test_fetch_config(config_file): """Verify fetch_config returns valid dictionnary""" config = resolve_config.fetch_config({"config": config_file}) assert config.pop('storage') == { 'database': { 'host': 'mongodb://*****:*****@localhost', 'name': 'orion_test', 'type': 'mongodb' } } assert config.pop('experiment') == { 'max_trials': 100, 'name': 'voila_voici', 'pool_size': 1, 'algorithms': 'random', 'strategy': 'NoParallelStrategy' } assert config == {}
def test_fetch_config_underscore(monkeypatch, config_file): """Verify fetch_config supports underscore as well.""" def mocked_config(file_object): return { 'experiment': { 'max_broken': 10, 'algorithms': { 'dont-change': 'me' } } } monkeypatch.setattr('yaml.safe_load', mocked_config) config = resolve_config.fetch_config({"config": config_file}) assert config == { 'experiment': { 'max_broken': 10, 'algorithms': { 'dont-change': 'me' } } }
def test_fetch_config_no_hit(): """Verify fetch_config returns empty dict on no config file path""" config = resolve_config.fetch_config({"config": ""}) assert config == {}
def fetch_file_config(self, cmdargs): """Get dictionary of options from configuration file provided in command-line""" return resolve_config.fetch_config(cmdargs)
def test_fetch_config_global_local_coherence(monkeypatch, config_file): """Verify fetch_config parses local config according to global config structure.""" def mocked_config(file_object): return orion.core.config.to_dict() monkeypatch.setattr("yaml.safe_load", mocked_config) config = resolve_config.fetch_config({"config": config_file}) # Test storage subconfig storage_config = config.pop("storage") database_config = storage_config.pop("database") assert storage_config.pop("type") == orion.core.config.storage.type assert storage_config == {} assert database_config.pop( "host") == orion.core.config.storage.database.host assert database_config.pop( "name") == orion.core.config.storage.database.name assert database_config.pop( "port") == orion.core.config.storage.database.port assert database_config.pop( "type") == orion.core.config.storage.database.type assert database_config == {} # Test experiment subconfig exp_config = config.pop("experiment") assert exp_config.pop( "max_trials") == orion.core.config.experiment.max_trials assert exp_config.pop( "max_broken") == orion.core.config.experiment.max_broken assert exp_config.pop( "working_dir") == orion.core.config.experiment.working_dir assert exp_config.pop( "pool_size") == orion.core.config.experiment.pool_size assert exp_config.pop( "algorithms") == orion.core.config.experiment.algorithms assert exp_config.pop("strategy") == orion.core.config.experiment.strategy assert exp_config == {} # Test worker subconfig worker_config = config.pop("worker") assert worker_config.pop("heartbeat") == orion.core.config.worker.heartbeat assert worker_config.pop( "max_trials") == orion.core.config.worker.max_trials assert worker_config.pop( "max_broken") == orion.core.config.worker.max_broken assert worker_config.pop( "max_idle_time") == orion.core.config.worker.max_idle_time assert (worker_config.pop("interrupt_signal_code") == orion.core.config.worker.interrupt_signal_code) assert (worker_config.pop("user_script_config") == orion.core.config.worker.user_script_config) assert worker_config == {} # Test evc subconfig evc_config = config.pop("evc") assert evc_config.pop( "auto_resolution") == orion.core.config.evc.auto_resolution assert (evc_config.pop("manual_resolution") == orion.core.config.evc.manual_resolution) assert (evc_config.pop("non_monitored_arguments") == orion.core.config.evc.non_monitored_arguments) assert (evc_config.pop("ignore_code_changes") == orion.core.config.evc.ignore_code_changes) assert evc_config.pop( "algorithm_change") == orion.core.config.evc.algorithm_change assert (evc_config.pop("orion_version_change") == orion.core.config.evc.orion_version_change) assert evc_config.pop( "code_change_type") == orion.core.config.evc.code_change_type assert evc_config.pop( "cli_change_type") == orion.core.config.evc.cli_change_type assert (evc_config.pop("config_change_type") == orion.core.config.evc.config_change_type) assert evc_config == {} # Confirm that all fields were tested. assert config == {}
def test_fetch_config_global_local_coherence(monkeypatch, config_file): """Verify fetch_config parses local config according to global config structure.""" def mocked_config(file_object): return orion.core.config.to_dict() monkeypatch.setattr('yaml.safe_load', mocked_config) config = resolve_config.fetch_config({"config": config_file}) # Test storage subconfig storage_config = config.pop('storage') database_config = storage_config.pop('database') assert storage_config.pop('type') == orion.core.config.storage.type assert storage_config == {} assert database_config.pop( 'host') == orion.core.config.storage.database.host assert database_config.pop( 'name') == orion.core.config.storage.database.name assert database_config.pop( 'port') == orion.core.config.storage.database.port assert database_config.pop( 'type') == orion.core.config.storage.database.type assert database_config == {} # Test experiment subconfig exp_config = config.pop('experiment') assert exp_config.pop( 'max_trials') == orion.core.config.experiment.max_trials assert exp_config.pop( 'max_broken') == orion.core.config.experiment.max_broken assert exp_config.pop( 'working_dir') == orion.core.config.experiment.working_dir assert exp_config.pop( 'pool_size') == orion.core.config.experiment.pool_size assert exp_config.pop( 'algorithms') == orion.core.config.experiment.algorithms assert exp_config.pop('strategy') == orion.core.config.experiment.strategy assert exp_config == {} # Test worker subconfig worker_config = config.pop('worker') assert worker_config.pop('heartbeat') == orion.core.config.worker.heartbeat assert worker_config.pop( 'max_trials') == orion.core.config.worker.max_trials assert worker_config.pop( 'max_broken') == orion.core.config.worker.max_broken assert worker_config.pop( 'max_idle_time') == orion.core.config.worker.max_idle_time assert (worker_config.pop('interrupt_signal_code') == orion.core.config.worker.interrupt_signal_code) assert (worker_config.pop('user_script_config') == orion.core.config.worker.user_script_config) assert worker_config == {} # Test evc subconfig evc_config = config.pop('evc') assert evc_config.pop( 'auto_resolution') == orion.core.config.evc.auto_resolution assert evc_config.pop( 'manual_resolution') == orion.core.config.evc.manual_resolution assert (evc_config.pop('non_monitored_arguments') == orion.core.config.evc.non_monitored_arguments) assert evc_config.pop( 'ignore_code_changes') == orion.core.config.evc.ignore_code_changes assert evc_config.pop( 'algorithm_change') == orion.core.config.evc.algorithm_change assert evc_config.pop( 'code_change_type') == orion.core.config.evc.code_change_type assert evc_config.pop( 'cli_change_type') == orion.core.config.evc.cli_change_type assert evc_config.pop( 'config_change_type') == orion.core.config.evc.config_change_type assert evc_config == {} # Confirm that all fields were tested. assert config == {}