def test_fetch_config_deprecate_tricky_names(monkeypatch, config_file):
"""Verify fetch_config assigns values properly for the similar names."""
def mocked_config(file_object):
return {
'experiment': {
'worker_trials': 'should_be_ignored'
},
'max_trials': 'exp_max_trials',
'max_broken': 'exp_max_broken',
'worker_trials': 'worker_max_trials',
'name': 'exp_name'
}
monkeypatch.setattr('yaml.safe_load', mocked_config)
config = resolve_config.fetch_config({"config": config_file})
assert config == {
'experiment': {
'name': 'exp_name',
'max_trials': 'exp_max_trials',
'max_broken': 'exp_max_broken'
},
'worker': {
'max_trials': 'worker_max_trials'
}
}
def test_fetch_config_deprecate_tricky_names(monkeypatch, config_file):
"""Verify fetch_config assigns values properly for the similar names."""
def mocked_config(file_object):
return {
"experiment": {
"worker_trials": "should_be_ignored"
},
"max_trials": "exp_max_trials",
"max_broken": "exp_max_broken",
"worker_trials": "worker_max_trials",
"name": "exp_name",
}
monkeypatch.setattr("yaml.safe_load", mocked_config)
config = resolve_config.fetch_config({"config": config_file})
assert config == {
"experiment": {
"name": "exp_name",
"max_trials": "exp_max_trials",
"max_broken": "exp_max_broken",
},
"worker": {
"max_trials": "worker_max_trials"
},
}
def test_fetch_config_deprecated_max_trials(monkeypatch, config_file):
"""Verify fetch_config will overwrite deprecated value if also properly defined."""
def mocked_config(file_object):
return {"experiment": {"max_trials": 10}, "max_trials": 20}
monkeypatch.setattr("yaml.safe_load", mocked_config)
config = resolve_config.fetch_config({"config": config_file})
assert config == {"experiment": {"max_trials": 10}}
def test_fetch_config(config_file):
"""Verify fetch_config returns valid dictionnary"""
config = resolve_config.fetch_config({"config": config_file})
assert config['algorithms'] == 'random'
assert config['database']['host'] == 'mongodb://*****:*****@localhost'
assert config['database']['name'] == 'orion_test'
assert config['database']['type'] == 'mongodb'
assert config['max_trials'] == 100
assert config['name'] == 'voila_voici'
assert config['pool_size'] == 1
def test_fetch_config_underscore(monkeypatch, config_file):
"""Verify fetch_config supports underscore as well."""
def mocked_config(file_object):
return {"experiment": {"max_broken": 10, "algorithms": {"dont-change": "me"}}}
monkeypatch.setattr("yaml.safe_load", mocked_config)
config = resolve_config.fetch_config({"config": config_file})
assert config == {
"experiment": {"max_broken": 10, "algorithms": {"dont-change": "me"}}
}
def get_cmd_config(cmdargs):
"""Fetch configuration defined by commandline and local configuration file.
Arguments of commandline have priority over options in configuration file.
"""
cmdargs = resolve_config.fetch_config_from_cmdargs(cmdargs)
cmd_config = resolve_config.fetch_config(cmdargs)
cmd_config = resolve_config.merge_configs(cmd_config, cmdargs)
cmd_config.update(cmd_config.pop("experiment", {}))
cmd_config["user_script_config"] = cmd_config.get("worker", {}).get(
"user_script_config", None
)
cmd_config["branching"] = cmd_config.pop("evc", {})
# TODO: We should move branching specific stuff below in a centralized place for EVC stuff.
if (
cmd_config["branching"].get("auto_resolution", False)
and cmdargs.get("manual_resolution", None) is None
):
cmd_config["branching"]["manual_resolution"] = False
non_monitored_arguments = cmdargs.get("non_monitored_arguments")
if non_monitored_arguments:
cmd_config["branching"][
"non_monitored_arguments"
] = non_monitored_arguments.split(":")
# TODO: user_args won't be defined if reading from DB only (`orion hunt -n <exp> ` alone)
metadata = resolve_config.fetch_metadata(
cmd_config.get("user"),
cmd_config.get("user_args"),
cmd_config.get("user_script_config"),
)
cmd_config["metadata"] = metadata
cmd_config.pop("config", None)
cmd_config["space"] = cmd_config["metadata"].get("priors", None)
backward.update_db_config(cmd_config)
return cmd_config
def test_fetch_config(config_file):
"""Verify fetch_config returns valid dictionnary"""
config = resolve_config.fetch_config({"config": config_file})
assert config.pop("storage") == {
"database": {
"host": "mongodb://*****:*****@localhost",
"name": "orion_test",
"type": "mongodb",
}
}
assert config.pop("experiment") == {
"max_trials": 100,
"max_broken": 5,
"name": "voila_voici",
"algorithms": "random",
}
assert config == {}
def get_cmd_config(cmdargs):
"""Fetch configuration defined by commandline and local configuration file.
Arguments of commandline have priority over options in configuration file.
"""
cmdargs = resolve_config.fetch_config_from_cmdargs(cmdargs)
cmd_config = resolve_config.fetch_config(cmdargs)
cmd_config = resolve_config.merge_configs(cmd_config, cmdargs)
cmd_config.update(cmd_config.pop('experiment', {}))
cmd_config['branching'] = cmd_config.pop('evc', {})
metadata = resolve_config.fetch_metadata(cmd_config.get('user'),
cmd_config.get('user_args'))
cmd_config['metadata'] = metadata
cmd_config.pop('config', None)
backward.populate_space(cmd_config)
backward.update_db_config(cmd_config)
return cmd_config
def test_fetch_config(config_file):
"""Verify fetch_config returns valid dictionnary"""
config = resolve_config.fetch_config({"config": config_file})
assert config.pop('storage') == {
'database': {
'host': 'mongodb://*****:*****@localhost',
'name': 'orion_test',
'type': 'mongodb'
}
}
assert config.pop('experiment') == {
'max_trials': 100,
'name': 'voila_voici',
'pool_size': 1,
'algorithms': 'random',
'strategy': 'NoParallelStrategy'
}
assert config == {}
def test_fetch_config_underscore(monkeypatch, config_file):
"""Verify fetch_config supports underscore as well."""
def mocked_config(file_object):
return {
'experiment': {
'max_broken': 10,
'algorithms': {
'dont-change': 'me'
}
}
}
monkeypatch.setattr('yaml.safe_load', mocked_config)
config = resolve_config.fetch_config({"config": config_file})
assert config == {
'experiment': {
'max_broken': 10,
'algorithms': {
'dont-change': 'me'
}
}
}
def test_fetch_config_no_hit():
"""Verify fetch_config returns empty dict on no config file path"""
config = resolve_config.fetch_config({"config": ""})
assert config == {}
def fetch_file_config(self, cmdargs):
"""Get dictionary of options from configuration file provided in command-line"""
return resolve_config.fetch_config(cmdargs)
def test_fetch_config_global_local_coherence(monkeypatch, config_file):
"""Verify fetch_config parses local config according to global config structure."""
def mocked_config(file_object):
return orion.core.config.to_dict()
monkeypatch.setattr("yaml.safe_load", mocked_config)
config = resolve_config.fetch_config({"config": config_file})
# Test storage subconfig
storage_config = config.pop("storage")
database_config = storage_config.pop("database")
assert storage_config.pop("type") == orion.core.config.storage.type
assert storage_config == {}
assert database_config.pop(
"host") == orion.core.config.storage.database.host
assert database_config.pop(
"name") == orion.core.config.storage.database.name
assert database_config.pop(
"port") == orion.core.config.storage.database.port
assert database_config.pop(
"type") == orion.core.config.storage.database.type
assert database_config == {}
# Test experiment subconfig
exp_config = config.pop("experiment")
assert exp_config.pop(
"max_trials") == orion.core.config.experiment.max_trials
assert exp_config.pop(
"max_broken") == orion.core.config.experiment.max_broken
assert exp_config.pop(
"working_dir") == orion.core.config.experiment.working_dir
assert exp_config.pop(
"pool_size") == orion.core.config.experiment.pool_size
assert exp_config.pop(
"algorithms") == orion.core.config.experiment.algorithms
assert exp_config.pop("strategy") == orion.core.config.experiment.strategy
assert exp_config == {}
# Test worker subconfig
worker_config = config.pop("worker")
assert worker_config.pop("heartbeat") == orion.core.config.worker.heartbeat
assert worker_config.pop(
"max_trials") == orion.core.config.worker.max_trials
assert worker_config.pop(
"max_broken") == orion.core.config.worker.max_broken
assert worker_config.pop(
"max_idle_time") == orion.core.config.worker.max_idle_time
assert (worker_config.pop("interrupt_signal_code") ==
orion.core.config.worker.interrupt_signal_code)
assert (worker_config.pop("user_script_config") ==
orion.core.config.worker.user_script_config)
assert worker_config == {}
# Test evc subconfig
evc_config = config.pop("evc")
assert evc_config.pop(
"auto_resolution") == orion.core.config.evc.auto_resolution
assert (evc_config.pop("manual_resolution") ==
orion.core.config.evc.manual_resolution)
assert (evc_config.pop("non_monitored_arguments") ==
orion.core.config.evc.non_monitored_arguments)
assert (evc_config.pop("ignore_code_changes") ==
orion.core.config.evc.ignore_code_changes)
assert evc_config.pop(
"algorithm_change") == orion.core.config.evc.algorithm_change
assert (evc_config.pop("orion_version_change") ==
orion.core.config.evc.orion_version_change)
assert evc_config.pop(
"code_change_type") == orion.core.config.evc.code_change_type
assert evc_config.pop(
"cli_change_type") == orion.core.config.evc.cli_change_type
assert (evc_config.pop("config_change_type") ==
orion.core.config.evc.config_change_type)
assert evc_config == {}
# Confirm that all fields were tested.
assert config == {}
def test_fetch_config_global_local_coherence(monkeypatch, config_file):
"""Verify fetch_config parses local config according to global config structure."""
def mocked_config(file_object):
return orion.core.config.to_dict()
monkeypatch.setattr('yaml.safe_load', mocked_config)
config = resolve_config.fetch_config({"config": config_file})
# Test storage subconfig
storage_config = config.pop('storage')
database_config = storage_config.pop('database')
assert storage_config.pop('type') == orion.core.config.storage.type
assert storage_config == {}
assert database_config.pop(
'host') == orion.core.config.storage.database.host
assert database_config.pop(
'name') == orion.core.config.storage.database.name
assert database_config.pop(
'port') == orion.core.config.storage.database.port
assert database_config.pop(
'type') == orion.core.config.storage.database.type
assert database_config == {}
# Test experiment subconfig
exp_config = config.pop('experiment')
assert exp_config.pop(
'max_trials') == orion.core.config.experiment.max_trials
assert exp_config.pop(
'max_broken') == orion.core.config.experiment.max_broken
assert exp_config.pop(
'working_dir') == orion.core.config.experiment.working_dir
assert exp_config.pop(
'pool_size') == orion.core.config.experiment.pool_size
assert exp_config.pop(
'algorithms') == orion.core.config.experiment.algorithms
assert exp_config.pop('strategy') == orion.core.config.experiment.strategy
assert exp_config == {}
# Test worker subconfig
worker_config = config.pop('worker')
assert worker_config.pop('heartbeat') == orion.core.config.worker.heartbeat
assert worker_config.pop(
'max_trials') == orion.core.config.worker.max_trials
assert worker_config.pop(
'max_broken') == orion.core.config.worker.max_broken
assert worker_config.pop(
'max_idle_time') == orion.core.config.worker.max_idle_time
assert (worker_config.pop('interrupt_signal_code') ==
orion.core.config.worker.interrupt_signal_code)
assert (worker_config.pop('user_script_config') ==
orion.core.config.worker.user_script_config)
assert worker_config == {}
# Test evc subconfig
evc_config = config.pop('evc')
assert evc_config.pop(
'auto_resolution') == orion.core.config.evc.auto_resolution
assert evc_config.pop(
'manual_resolution') == orion.core.config.evc.manual_resolution
assert (evc_config.pop('non_monitored_arguments') ==
orion.core.config.evc.non_monitored_arguments)
assert evc_config.pop(
'ignore_code_changes') == orion.core.config.evc.ignore_code_changes
assert evc_config.pop(
'algorithm_change') == orion.core.config.evc.algorithm_change
assert evc_config.pop(
'code_change_type') == orion.core.config.evc.code_change_type
assert evc_config.pop(
'cli_change_type') == orion.core.config.evc.cli_change_type
assert evc_config.pop(
'config_change_type') == orion.core.config.evc.config_change_type
assert evc_config == {}
# Confirm that all fields were tested.
assert config == {}
