def test_format_commandline_only(
parser: OrionCmdlineParser, commandline: List[str], weird_argument: WeirdArgument
):
"""Format the commandline using only args."""
parser.parse(commandline)
trial = Trial(
params=[
{"name": "/lr", "type": "real", "value": -2.4},
{"name": "/prior", "type": "categorical", "value": "sgd"},
{"name": "/a.b", "type": "real", "value": 0.5},
{
"name": f"/{weird_argument.name}",
"type": weird_argument.prior_type,
"value": weird_argument.value,
},
]
)
cmd_inst = parser.format(trial=trial)
assert cmd_inst == [
"--seed",
"555",
"--lr",
"-2.4",
"--non-prior",
"choices({'sgd': 0.2, 'adam': 0.8})",
"--prior",
"sgd",
"--a.b",
"0.5",
f"--{weird_argument.name}",
f"{weird_argument.value}",
]
def __init__(
self,
experiment,
user_script_config=None,
interrupt_signal_code=None,
ignore_code_changes=None,
):
log.debug("Creating Consumer object.")
self.experiment = experiment
self.space = experiment.space
if self.space is None:
raise RuntimeError(
"Experiment object provided to Consumer has not yet completed"
" initialization."
)
if user_script_config is None:
user_script_config = orion.core.config.worker.user_script_config
if interrupt_signal_code is None:
interrupt_signal_code = orion.core.config.worker.interrupt_signal_code
# NOTE: If ignore_code_changes is None, we can assume EVC is enabled.
if ignore_code_changes is None:
ignore_code_changes = orion.core.config.evc.ignore_code_changes
self.interrupt_signal_code = interrupt_signal_code
self.ignore_code_changes = ignore_code_changes
# Fetch space builder
self.template_builder = OrionCmdlineParser(user_script_config)
self.template_builder.set_state_dict(experiment.metadata["parser"])
self.pacemaker = None
def __init__(self, experiment):
"""Initialize a consumer.
:param experiment: Manager of this experiment, provides convenient
interface for interacting with the database.
"""
log.debug("Creating Consumer object.")
self.experiment = experiment
self.space = experiment.space
if self.space is None:
raise RuntimeError(
"Experiment object provided to Consumer has not yet completed"
" initialization.")
# Fetch space builder
self.template_builder = OrionCmdlineParser(
orion.core.config.user_script_config)
self.template_builder.set_state_dict(experiment.metadata['parser'])
# Get path to user's script and infer trial configuration directory
if experiment.working_dir:
self.working_dir = os.path.abspath(experiment.working_dir)
else:
self.working_dir = os.path.join(tempfile.gettempdir(), 'orion')
self.script_path = experiment.metadata['user_script']
self.pacemaker = None
def test_format_with_properties(
parser: OrionCmdlineParser,
cmd_with_properties: List[str],
hacked_exp: Experiment,
weird_argument: WeirdArgument,
):
"""Test if format correctly puts the value of `trial` and `exp` when used as properties"""
parser.parse(cmd_with_properties)
# NOTE: Also using a weird argument here, to make sure the parser is able to distinguish
# property look-up vs weird argument names.
trial = Trial(
experiment="trial_test",
params=[
{"name": "/lr", "type": "real", "value": -2.4},
{"name": "/prior", "type": "categorical", "value": "sgd"},
{
"name": f"/{weird_argument.name}",
"type": weird_argument.prior_type,
"value": weird_argument.value,
},
],
)
cmd_line = parser.format(None, trial=trial, experiment=hacked_exp)
assert trial.hash_name in cmd_line
assert "supernaedo2-dendi" in cmd_line
def fetch_metadata(user=None, user_args=None, user_script_config=None):
"""Infer rest information about the process + versioning"""
metadata = {"user": user if user else getpass.getuser()}
metadata["orion_version"] = orion.core.__version__
if user_args is None:
user_args = []
# Trailing white space are catched by argparse as an empty argument
if len(user_args) == 1 and user_args[0] == "":
user_args = []
if user_script_config is None:
user_script_config = orion.core.config.worker.user_script_config
cmdline_parser = OrionCmdlineParser(user_script_config)
cmdline_parser.parse(user_args)
if cmdline_parser.user_script:
# TODO: Remove this, it is all in cmdline_parser now
metadata["user_script"] = cmdline_parser.user_script
metadata["VCS"] = infer_versioning_metadata(cmdline_parser.user_script)
if user_args:
metadata["user_args"] = user_args
metadata["parser"] = cmdline_parser.get_state_dict()
metadata["user_script_config"] = user_script_config
metadata["priors"] = dict(cmdline_parser.priors)
return metadata
def build_from_config(self, config):
"""Build a fully configured (and writable) experiment based on full configuration.
.. seealso::
`orion.core.io.experiment_builder` for more information on the hierarchy of
configurations.
:class:`orion.core.worker.experiment.Experiment` for more information on the experiment
object.
"""
log.info(config)
# Pop out configuration concerning databases and resources
config.pop('database', None)
config.pop('resources', None)
experiment = Experiment(config['name'], config.get('user', None),
config.get('version', None))
# TODO: Handle both from cmdline and python APIs.
if 'priors' not in config['metadata'] and 'user_args' not in config['metadata']:
raise NoConfigurationError
# Parse to generate priors
if 'user_args' in config['metadata']:
parser = OrionCmdlineParser(orion.core.config.user_script_config)
parser.parse(config['metadata']['user_args'])
config['metadata']['parser'] = parser.get_state_dict()
config['metadata']['priors'] = dict(parser.priors)
# Finish experiment's configuration and write it to database.
experiment.configure(config)
return experiment
def fetch_metadata(user=None, user_args=None):
"""Infer rest information about the process + versioning"""
metadata = {'user': user if user else getpass.getuser()}
metadata['orion_version'] = orion.core.__version__
if user_args is None:
user_args = []
# Trailing white space are catched by argparse as an empty argument
if len(user_args) == 1 and user_args[0] == '':
user_args = []
cmdline_parser = OrionCmdlineParser(config.worker.user_script_config)
cmdline_parser.parse(user_args)
if cmdline_parser.user_script:
# TODO: Remove this, it is all in cmdline_parser now
metadata['user_script'] = cmdline_parser.user_script
metadata['VCS'] = infer_versioning_metadata(cmdline_parser.user_script)
if user_args:
# TODO: Remove this, it is all in cmdline_parser now
metadata['user_args'] = user_args
return metadata
def __init__(self, experiment, heartbeat=None, user_script_config=None,
interrupt_signal_code=None):
log.debug("Creating Consumer object.")
self.experiment = experiment
self.space = experiment.space
if self.space is None:
raise RuntimeError("Experiment object provided to Consumer has not yet completed"
" initialization.")
if heartbeat is None:
heartbeat = orion.core.config.worker.heartbeat
if user_script_config is None:
user_script_config = orion.core.config.worker.user_script_config
if interrupt_signal_code is None:
interrupt_signal_code = orion.core.config.worker.interrupt_signal_code
self.heartbeat = heartbeat
self.interrupt_signal_code = interrupt_signal_code
# Fetch space builder
self.template_builder = OrionCmdlineParser(user_script_config)
self.template_builder.set_state_dict(experiment.metadata['parser'])
# Get path to user's script and infer trial configuration directory
if experiment.working_dir:
self.working_dir = os.path.abspath(experiment.working_dir)
else:
self.working_dir = os.path.join(tempfile.gettempdir(), 'orion')
self.pacemaker = None
def populate_priors(metadata):
"""Compute parser state and priors based on user_args and populate metadata."""
if 'user_args' not in metadata:
return
parser = OrionCmdlineParser(orion.core.config.user_script_config)
parser.parse(metadata["user_args"])
metadata["parser"] = parser.get_state_dict()
metadata["priors"] = dict(parser.priors)
def test_parse_finds_conflict(parser: OrionCmdlineParser,
commandline: List[str], yaml_config: List[str]):
"""Parse find conflicting declaration in commandline and config file."""
cmd_args = yaml_config
cmd_args.extend(commandline)
cmd_args.extend(["--something-same~choices({'sgd': 0.2, 'adam': 0.8})"])
with pytest.raises(ValueError) as exc:
parser.parse(cmd_args)
assert "Conflict" in str(exc.value)
def populate_priors(metadata):
"""Compute parser state and priors based on user_args and populate metadata."""
if 'user_args' not in metadata:
return
update_user_args(metadata)
parser = OrionCmdlineParser(orion.core.config.worker.user_script_config,
allow_non_existing_user_script=True)
parser.parse(metadata["user_args"])
metadata["parser"] = parser.get_state_dict()
metadata["priors"] = dict(parser.priors)
def test_parse_from_json_config(parser: OrionCmdlineParser, json_config: List[str]):
"""Parse from a json config only."""
parser.parse(json_config)
config = parser.priors
assert len(config.keys()) == 6
assert "/layers/1/width" in config
assert "/layers/1/type" in config
assert "/layers/2/type" in config
assert "/training/lr0" in config
assert "/training/mbs" in config
assert "/something-same" in config
def test_configurable_config_arg(parser_diff_prefix: OrionCmdlineParser,
yaml_sample_path: str):
"""Parse from a yaml config only."""
parser_diff_prefix.parse(["--config2", yaml_sample_path])
config = parser_diff_prefix.priors
assert len(config.keys()) == 6
assert "/layers/1/width" in config
assert "/layers/1/type" in config
assert "/layers/2/type" in config
assert "/training/lr0" in config
assert "/training/mbs" in config
assert "/something-same" in config
def test_get_state_dict_after_parse_no_config_file(parser: OrionCmdlineParser,
commandline: List[str]):
"""Test getting state dict."""
parser.parse(commandline)
assert parser.get_state_dict() == {
"parser": parser.parser.get_state_dict(),
"cmd_priors": list(map(list, parser.cmd_priors.items())),
"file_priors": list(map(list, parser.file_priors.items())),
"config_file_data": parser.config_file_data,
"config_prefix": parser.config_prefix,
"file_config_path": parser.file_config_path,
"converter": None,
}
def test_get_state_dict_after_parse_no_config_file(commandline):
"""Test getting state dict."""
parser = OrionCmdlineParser()
parser.parse(commandline)
assert parser.get_state_dict() == {
'parser': parser.parser.get_state_dict(),
'cmd_priors': list(map(list, parser.cmd_priors.items())),
'file_priors': list(map(list, parser.file_priors.items())),
'config_file_data': parser.config_file_data,
'config_prefix': parser.config_prefix,
'file_config_path': parser.file_config_path,
'converter': None
}
def build_from(self, config):
"""Build a `Space` object from a configuration.
Initialize a new parser for this commandline and parse the given config then
build a `Space` object from that configuration.
Returns
-------
`orion.algo.space.Space`
The problem's search space definition.
"""
self.parser = OrionCmdlineParser(orion_config.user_script_config)
self.parser.parse(config)
return self.build(self.parser.priors)
def test_get_state_dict_before_parse(commandline):
"""Test getting state dict."""
parser = OrionCmdlineParser()
assert parser.get_state_dict() == {
'parser': {
'arguments': [],
'template': []
},
'cmd_priors': list(map(list, parser.cmd_priors.items())),
'file_priors': list(map(list, parser.file_priors.items())),
'config_file_data': parser.config_file_data,
'config_prefix': parser.config_prefix,
'file_config_path': parser.file_config_path,
'converter': None
}
def update_metadata(metadata):
"""Update information about the process + versioning"""
metadata.setdefault("user", getpass.getuser())
metadata["orion_version"] = orion.core.__version__
if not metadata.get("user_args"):
return metadata
cmdline_parser = OrionCmdlineParser()
cmdline_parser.set_state_dict(metadata["parser"])
if cmdline_parser.user_script:
# TODO: Remove this, it is all in cmdline_parser now
metadata["user_script"] = cmdline_parser.user_script
metadata["VCS"] = infer_versioning_metadata(cmdline_parser.user_script)
return metadata
def test_parse_equivalency(yaml_config, json_config):
"""Templates found from json and yaml are the same."""
parser_yaml = OrionCmdlineParser()
parser_yaml.parse(yaml_config)
dict_from_yaml = parser_yaml.config_file_data
parser_json = OrionCmdlineParser()
parser_json.parse(json_config)
dict_from_json = parser_json.config_file_data
assert dict_from_json == dict_from_yaml
def test_configurable_config_arg_do_not_exist(script_path: str):
"""Test that parser can handle command if config file does not exist"""
parser = OrionCmdlineParser()
command = f"python {script_path} --config idontexist.yaml".split(" ")
with pytest.raises(OSError) as exc:
parser.parse(command)
assert exc.match("The path specified for the script config does not exist")
parser = OrionCmdlineParser(allow_non_existing_files=True)
parser.parse(command)
def test_parse_equivalency(yaml_config: List[str], json_config: List[str]):
"""Templates found from json and yaml are the same."""
parser_yaml = OrionCmdlineParser(allow_non_existing_files=True)
parser_yaml.parse(yaml_config)
dict_from_yaml = parser_yaml.config_file_data
parser_json = OrionCmdlineParser(allow_non_existing_files=True)
parser_json.parse(json_config)
dict_from_json = parser_json.config_file_data
assert dict_from_json == dict_from_yaml
def test_get_state_dict_after_parse_with_config_file(
parser: OrionCmdlineParser, yaml_config: List[str], commandline: List[str]
):
"""Test getting state dict."""
cmd_args = yaml_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
assert parser.get_state_dict() == {
"parser": parser.parser.get_state_dict(),
"cmd_priors": list(map(list, parser.cmd_priors.items())),
"file_priors": list(map(list, parser.file_priors.items())),
"config_file_data": parser.config_file_data,
"config_prefix": parser.config_prefix,
"file_config_path": parser.file_config_path,
"converter": parser.converter.get_state_dict(),
}
def test_get_state_dict_after_parse_with_config_file(yaml_config, commandline):
"""Test getting state dict."""
parser = OrionCmdlineParser()
cmd_args = yaml_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
assert parser.get_state_dict() == {
'parser': parser.parser.get_state_dict(),
'cmd_priors': list(map(list, parser.cmd_priors.items())),
'file_priors': list(map(list, parser.file_priors.items())),
'config_file_data': parser.config_file_data,
'config_prefix': parser.config_prefix,
'file_config_path': parser.file_config_path,
'converter': parser.converter.get_state_dict()
}
def test_infer_user_script_when_missing():
"""Test that user script is infered correctly even if missing"""
parser = OrionCmdlineParser()
with pytest.raises(FileNotFoundError) as exc:
parser.parse("python script.py and some args".split(" "))
assert exc.match("The path specified for the script does not exist")
parser = OrionCmdlineParser(allow_non_existing_files=True)
parser.parse("python script.py and some args".split(" "))
assert parser.user_script == "script.py"
def test_parse_from_args_and_config_yaml(
parser: OrionCmdlineParser,
commandline: List[str],
yaml_config: List[str],
weird_argument: WeirdArgument,
):
"""Parse both from commandline and config file."""
cmd_args = yaml_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
config = parser.priors
assert len(config) == 10
assert "/lr" in config
assert "/prior" in config
assert "/layers/1/width" in config
assert "/layers/1/type" in config
assert "/layers/2/type" in config
assert "/training/lr0" in config
assert "/training/mbs" in config
assert "/something-same" in config
assert "/a.b" in config
assert f"/{weird_argument.name}" in config
template = parser.parser.template
assert template == [
"--config",
"{config}",
"--seed",
"{seed}",
"--lr",
"{lr}",
"--non-prior",
"{non-prior}",
"--prior",
"{prior}",
"--a.b",
"{a.b}",
f"--{weird_argument.name}",
f"{{{weird_argument.name}}}",
]
def update_dropout(experiment_config):
metadata = experiment_config["metadata"]
user_script = metadata.get("user_script", "")
user_args = metadata.get("user_args", [])
try:
index = user_args.index("--dropout")
except ValueError:
print(
f"No dropout for {experiment_config['metadata']}-v{experiment_config['version']}"
)
return
user_args[index + 1] = (user_args[index + 1].replace("5,", "0.5,").replace(
", discrete=True", ", precision=1"))
cmdline_parser = OrionCmdlineParser(allow_non_existing_files=True)
cmdline_parser.parse([user_script] + user_args)
metadata["parser"] = cmdline_parser.get_state_dict()
experiment_config["space"] = metadata["priors"] = dict(
cmdline_parser.priors)
# Update config in db
storage.update_experiment(uid=experiment_config["_id"],
**experiment_config)
# Update all trials in db (arf)
n_trials_before = len(storage.fetch_trials(uid=experiment_config["_id"]))
for trial in storage.fetch_trials(uid=experiment_config["_id"]):
previous_id = trial.id
for param in trial._params:
if param.name == "/dropout":
param.value /= 10
assert 0 <= param.value <= 0.5, param.value
storage.delete_trials(uid=experiment_config["_id"],
where=dict(_id=previous_id))
storage.register_trial(trial)
trials = storage.fetch_trials(uid=experiment_config["_id"])
assert len(trials) == n_trials_before, len(trials)
for trial in trials:
assert 0 <= trial.params["/dropout"] <= 0.5, trial
def test_get_state_dict_before_parse(
parser: OrionCmdlineParser, commandline: List[str]
):
"""Test getting state dict."""
assert parser.get_state_dict() == {
"parser": {"keys": [], "arguments": [], "template": []},
"cmd_priors": list(map(list, parser.cmd_priors.items())),
"file_priors": list(map(list, parser.file_priors.items())),
"config_file_data": parser.config_file_data,
"config_prefix": parser.config_prefix,
"file_config_path": parser.file_config_path,
"converter": None,
}
def test_format_without_config_path(
parser: OrionCmdlineParser,
commandline: List[str],
json_config: List[str],
tmp_path: Path,
json_converter: JSONConverter,
weird_argument: WeirdArgument,
):
"""Verify that parser.format() raises ValueError when config path not passed."""
cmd_args = json_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
trial = Trial(
params=[
{"name": "/lr", "type": "real", "value": -2.4},
{"name": "/prior", "type": "categorical", "value": "sgd"},
{"name": "/layers/1/width", "type": "integer", "value": 100},
{"name": "/layers/1/type", "type": "categorical", "value": "relu"},
{"name": "/layers/2/type", "type": "categorical", "value": "sigmoid"},
{"name": "/training/lr0", "type": "real", "value": 0.032},
{"name": "/training/mbs", "type": "integer", "value": 64},
{"name": "/something-same", "type": "categorical", "value": "3"},
{
"name": f"/{weird_argument.name}",
"type": "categorical",
"value": weird_argument.value,
},
]
)
with pytest.raises(
ValueError, match="Cannot format without a `config_path` argument."
):
parser.format(trial=trial)
def test_parse_from_args_only(
parser: OrionCmdlineParser,
commandline_fluff: List[str],
weird_argument: WeirdArgument,
):
"""Parse a commandline."""
cmd_args = commandline_fluff
parser.parse(cmd_args)
assert not parser.config_file_data
assert len(parser.cmd_priors) == 4
assert "/lr" in parser.cmd_priors
assert "/prior" in parser.cmd_priors
assert "/a.b" in parser.cmd_priors
assert f"/{weird_argument.name}" in parser.cmd_priors
assert parser.parser.template == [
"--seed",
"{seed}",
"--lr",
"{lr}",
"--non-prior",
"{non-prior}",
"--prior",
"{prior}",
"--a.b",
"{a.b}",
f"--{weird_argument.name}",
f"{{{weird_argument.name}}}",
"--some-path",
"{some-path}",
"--home-path",
"{home-path[0]}",
"{home-path[1]}",
]
def populate_priors(metadata):
"""Compute parser state and priors based on user_args and populate metadata."""
if "user_args" not in metadata:
return
update_user_args(metadata)
parser = OrionCmdlineParser(orion.core.config.worker.user_script_config,
allow_non_existing_files=True)
if "parser" in metadata:
# To keep configs like config user_script_config
parser.config_prefix = metadata["parser"]["config_prefix"]
parser.parse(metadata["user_args"])
log.debug("Updating parser for backward compatibility")
metadata["parser"] = parser.get_state_dict()
log.debug(pprint.pformat(metadata["parser"]))
log.debug("Updating priors for backward compatibility")
metadata["priors"] = dict(parser.priors)
log.debug(pprint.pformat(metadata["priors"]))