def fetch_metadata(user=None, user_args=None, user_script_config=None):
"""Infer rest information about the process + versioning"""
metadata = {"user": user if user else getpass.getuser()}
metadata["orion_version"] = orion.core.__version__
if user_args is None:
user_args = []
# Trailing white space are catched by argparse as an empty argument
if len(user_args) == 1 and user_args[0] == "":
user_args = []
if user_script_config is None:
user_script_config = orion.core.config.worker.user_script_config
cmdline_parser = OrionCmdlineParser(user_script_config)
cmdline_parser.parse(user_args)
if cmdline_parser.user_script:
# TODO: Remove this, it is all in cmdline_parser now
metadata["user_script"] = cmdline_parser.user_script
metadata["VCS"] = infer_versioning_metadata(cmdline_parser.user_script)
if user_args:
metadata["user_args"] = user_args
metadata["parser"] = cmdline_parser.get_state_dict()
metadata["user_script_config"] = user_script_config
metadata["priors"] = dict(cmdline_parser.priors)
return metadata
def build_from_config(self, config):
"""Build a fully configured (and writable) experiment based on full configuration.
.. seealso::
`orion.core.io.experiment_builder` for more information on the hierarchy of
configurations.
:class:`orion.core.worker.experiment.Experiment` for more information on the experiment
object.
"""
log.info(config)
# Pop out configuration concerning databases and resources
config.pop('database', None)
config.pop('resources', None)
experiment = Experiment(config['name'], config.get('user', None),
config.get('version', None))
# TODO: Handle both from cmdline and python APIs.
if 'priors' not in config['metadata'] and 'user_args' not in config['metadata']:
raise NoConfigurationError
# Parse to generate priors
if 'user_args' in config['metadata']:
parser = OrionCmdlineParser(orion.core.config.user_script_config)
parser.parse(config['metadata']['user_args'])
config['metadata']['parser'] = parser.get_state_dict()
config['metadata']['priors'] = dict(parser.priors)
# Finish experiment's configuration and write it to database.
experiment.configure(config)
return experiment
def test_format_commandline_only(
parser: OrionCmdlineParser, commandline: List[str], weird_argument: WeirdArgument
):
"""Format the commandline using only args."""
parser.parse(commandline)
trial = Trial(
params=[
{"name": "/lr", "type": "real", "value": -2.4},
{"name": "/prior", "type": "categorical", "value": "sgd"},
{"name": "/a.b", "type": "real", "value": 0.5},
{
"name": f"/{weird_argument.name}",
"type": weird_argument.prior_type,
"value": weird_argument.value,
},
]
)
cmd_inst = parser.format(trial=trial)
assert cmd_inst == [
"--seed",
"555",
"--lr",
"-2.4",
"--non-prior",
"choices({'sgd': 0.2, 'adam': 0.8})",
"--prior",
"sgd",
"--a.b",
"0.5",
f"--{weird_argument.name}",
f"{weird_argument.value}",
]
def fetch_metadata(user=None, user_args=None):
"""Infer rest information about the process + versioning"""
metadata = {'user': user if user else getpass.getuser()}
metadata['orion_version'] = orion.core.__version__
if user_args is None:
user_args = []
# Trailing white space are catched by argparse as an empty argument
if len(user_args) == 1 and user_args[0] == '':
user_args = []
cmdline_parser = OrionCmdlineParser(config.worker.user_script_config)
cmdline_parser.parse(user_args)
if cmdline_parser.user_script:
# TODO: Remove this, it is all in cmdline_parser now
metadata['user_script'] = cmdline_parser.user_script
metadata['VCS'] = infer_versioning_metadata(cmdline_parser.user_script)
if user_args:
# TODO: Remove this, it is all in cmdline_parser now
metadata['user_args'] = user_args
return metadata
def test_format_with_properties(
parser: OrionCmdlineParser,
cmd_with_properties: List[str],
hacked_exp: Experiment,
weird_argument: WeirdArgument,
):
"""Test if format correctly puts the value of `trial` and `exp` when used as properties"""
parser.parse(cmd_with_properties)
# NOTE: Also using a weird argument here, to make sure the parser is able to distinguish
# property look-up vs weird argument names.
trial = Trial(
experiment="trial_test",
params=[
{"name": "/lr", "type": "real", "value": -2.4},
{"name": "/prior", "type": "categorical", "value": "sgd"},
{
"name": f"/{weird_argument.name}",
"type": weird_argument.prior_type,
"value": weird_argument.value,
},
],
)
cmd_line = parser.format(None, trial=trial, experiment=hacked_exp)
assert trial.hash_name in cmd_line
assert "supernaedo2-dendi" in cmd_line
def populate_priors(metadata):
"""Compute parser state and priors based on user_args and populate metadata."""
if 'user_args' not in metadata:
return
parser = OrionCmdlineParser(orion.core.config.user_script_config)
parser.parse(metadata["user_args"])
metadata["parser"] = parser.get_state_dict()
metadata["priors"] = dict(parser.priors)
def test_parse_equivalency(yaml_config: List[str], json_config: List[str]):
"""Templates found from json and yaml are the same."""
parser_yaml = OrionCmdlineParser(allow_non_existing_files=True)
parser_yaml.parse(yaml_config)
dict_from_yaml = parser_yaml.config_file_data
parser_json = OrionCmdlineParser(allow_non_existing_files=True)
parser_json.parse(json_config)
dict_from_json = parser_json.config_file_data
assert dict_from_json == dict_from_yaml
def test_configurable_config_arg_do_not_exist(script_path: str):
"""Test that parser can handle command if config file does not exist"""
parser = OrionCmdlineParser()
command = f"python {script_path} --config idontexist.yaml".split(" ")
with pytest.raises(OSError) as exc:
parser.parse(command)
assert exc.match("The path specified for the script config does not exist")
parser = OrionCmdlineParser(allow_non_existing_files=True)
parser.parse(command)
def test_parse_equivalency(yaml_config, json_config):
"""Templates found from json and yaml are the same."""
parser_yaml = OrionCmdlineParser()
parser_yaml.parse(yaml_config)
dict_from_yaml = parser_yaml.config_file_data
parser_json = OrionCmdlineParser()
parser_json.parse(json_config)
dict_from_json = parser_json.config_file_data
assert dict_from_json == dict_from_yaml
def test_parse_finds_conflict(parser: OrionCmdlineParser,
commandline: List[str], yaml_config: List[str]):
"""Parse find conflicting declaration in commandline and config file."""
cmd_args = yaml_config
cmd_args.extend(commandline)
cmd_args.extend(["--something-same~choices({'sgd': 0.2, 'adam': 0.8})"])
with pytest.raises(ValueError) as exc:
parser.parse(cmd_args)
assert "Conflict" in str(exc.value)
def test_infer_user_script_when_missing():
"""Test that user script is infered correctly even if missing"""
parser = OrionCmdlineParser()
with pytest.raises(FileNotFoundError) as exc:
parser.parse("python script.py and some args".split(" "))
assert exc.match("The path specified for the script does not exist")
parser = OrionCmdlineParser(allow_non_existing_files=True)
parser.parse("python script.py and some args".split(" "))
assert parser.user_script == "script.py"
def populate_priors(metadata):
"""Compute parser state and priors based on user_args and populate metadata."""
if 'user_args' not in metadata:
return
update_user_args(metadata)
parser = OrionCmdlineParser(orion.core.config.worker.user_script_config,
allow_non_existing_user_script=True)
parser.parse(metadata["user_args"])
metadata["parser"] = parser.get_state_dict()
metadata["priors"] = dict(parser.priors)
def test_parse_from_json_config(parser: OrionCmdlineParser, json_config: List[str]):
"""Parse from a json config only."""
parser.parse(json_config)
config = parser.priors
assert len(config.keys()) == 6
assert "/layers/1/width" in config
assert "/layers/1/type" in config
assert "/layers/2/type" in config
assert "/training/lr0" in config
assert "/training/mbs" in config
assert "/something-same" in config
def test_configurable_config_arg(parser_diff_prefix: OrionCmdlineParser,
yaml_sample_path: str):
"""Parse from a yaml config only."""
parser_diff_prefix.parse(["--config2", yaml_sample_path])
config = parser_diff_prefix.priors
assert len(config.keys()) == 6
assert "/layers/1/width" in config
assert "/layers/1/type" in config
assert "/layers/2/type" in config
assert "/training/lr0" in config
assert "/training/mbs" in config
assert "/something-same" in config
def test_get_state_dict_after_parse_no_config_file(parser: OrionCmdlineParser,
commandline: List[str]):
"""Test getting state dict."""
parser.parse(commandline)
assert parser.get_state_dict() == {
"parser": parser.parser.get_state_dict(),
"cmd_priors": list(map(list, parser.cmd_priors.items())),
"file_priors": list(map(list, parser.file_priors.items())),
"config_file_data": parser.config_file_data,
"config_prefix": parser.config_prefix,
"file_config_path": parser.file_config_path,
"converter": None,
}
def test_get_state_dict_after_parse_no_config_file(commandline):
"""Test getting state dict."""
parser = OrionCmdlineParser()
parser.parse(commandline)
assert parser.get_state_dict() == {
'parser': parser.parser.get_state_dict(),
'cmd_priors': list(map(list, parser.cmd_priors.items())),
'file_priors': list(map(list, parser.file_priors.items())),
'config_file_data': parser.config_file_data,
'config_prefix': parser.config_prefix,
'file_config_path': parser.file_config_path,
'converter': None
}
def test_get_state_dict_after_parse_with_config_file(
parser: OrionCmdlineParser, yaml_config: List[str], commandline: List[str]
):
"""Test getting state dict."""
cmd_args = yaml_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
assert parser.get_state_dict() == {
"parser": parser.parser.get_state_dict(),
"cmd_priors": list(map(list, parser.cmd_priors.items())),
"file_priors": list(map(list, parser.file_priors.items())),
"config_file_data": parser.config_file_data,
"config_prefix": parser.config_prefix,
"file_config_path": parser.file_config_path,
"converter": parser.converter.get_state_dict(),
}
def test_get_state_dict_after_parse_with_config_file(yaml_config, commandline):
"""Test getting state dict."""
parser = OrionCmdlineParser()
cmd_args = yaml_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
assert parser.get_state_dict() == {
'parser': parser.parser.get_state_dict(),
'cmd_priors': list(map(list, parser.cmd_priors.items())),
'file_priors': list(map(list, parser.file_priors.items())),
'config_file_data': parser.config_file_data,
'config_prefix': parser.config_prefix,
'file_config_path': parser.file_config_path,
'converter': parser.converter.get_state_dict()
}
def test_parse_from_args_and_config_yaml(
parser: OrionCmdlineParser,
commandline: List[str],
yaml_config: List[str],
weird_argument: WeirdArgument,
):
"""Parse both from commandline and config file."""
cmd_args = yaml_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
config = parser.priors
assert len(config) == 10
assert "/lr" in config
assert "/prior" in config
assert "/layers/1/width" in config
assert "/layers/1/type" in config
assert "/layers/2/type" in config
assert "/training/lr0" in config
assert "/training/mbs" in config
assert "/something-same" in config
assert "/a.b" in config
assert f"/{weird_argument.name}" in config
template = parser.parser.template
assert template == [
"--config",
"{config}",
"--seed",
"{seed}",
"--lr",
"{lr}",
"--non-prior",
"{non-prior}",
"--prior",
"{prior}",
"--a.b",
"{a.b}",
f"--{weird_argument.name}",
f"{{{weird_argument.name}}}",
]
def update_dropout(experiment_config):
metadata = experiment_config["metadata"]
user_script = metadata.get("user_script", "")
user_args = metadata.get("user_args", [])
try:
index = user_args.index("--dropout")
except ValueError:
print(
f"No dropout for {experiment_config['metadata']}-v{experiment_config['version']}"
)
return
user_args[index + 1] = (user_args[index + 1].replace("5,", "0.5,").replace(
", discrete=True", ", precision=1"))
cmdline_parser = OrionCmdlineParser(allow_non_existing_files=True)
cmdline_parser.parse([user_script] + user_args)
metadata["parser"] = cmdline_parser.get_state_dict()
experiment_config["space"] = metadata["priors"] = dict(
cmdline_parser.priors)
# Update config in db
storage.update_experiment(uid=experiment_config["_id"],
**experiment_config)
# Update all trials in db (arf)
n_trials_before = len(storage.fetch_trials(uid=experiment_config["_id"]))
for trial in storage.fetch_trials(uid=experiment_config["_id"]):
previous_id = trial.id
for param in trial._params:
if param.name == "/dropout":
param.value /= 10
assert 0 <= param.value <= 0.5, param.value
storage.delete_trials(uid=experiment_config["_id"],
where=dict(_id=previous_id))
storage.register_trial(trial)
trials = storage.fetch_trials(uid=experiment_config["_id"])
assert len(trials) == n_trials_before, len(trials)
for trial in trials:
assert 0 <= trial.params["/dropout"] <= 0.5, trial
def populate_priors(metadata):
"""Compute parser state and priors based on user_args and populate metadata."""
if "user_args" not in metadata:
return
update_user_args(metadata)
parser = OrionCmdlineParser(orion.core.config.worker.user_script_config,
allow_non_existing_files=True)
if "parser" in metadata:
# To keep configs like config user_script_config
parser.config_prefix = metadata["parser"]["config_prefix"]
parser.parse(metadata["user_args"])
log.debug("Updating parser for backward compatibility")
metadata["parser"] = parser.get_state_dict()
log.debug(pprint.pformat(metadata["parser"]))
log.debug("Updating priors for backward compatibility")
metadata["priors"] = dict(parser.priors)
log.debug(pprint.pformat(metadata["priors"]))
def test_format_without_config_path(
parser: OrionCmdlineParser,
commandline: List[str],
json_config: List[str],
tmp_path: Path,
json_converter: JSONConverter,
weird_argument: WeirdArgument,
):
"""Verify that parser.format() raises ValueError when config path not passed."""
cmd_args = json_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
trial = Trial(
params=[
{"name": "/lr", "type": "real", "value": -2.4},
{"name": "/prior", "type": "categorical", "value": "sgd"},
{"name": "/layers/1/width", "type": "integer", "value": 100},
{"name": "/layers/1/type", "type": "categorical", "value": "relu"},
{"name": "/layers/2/type", "type": "categorical", "value": "sigmoid"},
{"name": "/training/lr0", "type": "real", "value": 0.032},
{"name": "/training/mbs", "type": "integer", "value": 64},
{"name": "/something-same", "type": "categorical", "value": "3"},
{
"name": f"/{weird_argument.name}",
"type": "categorical",
"value": weird_argument.value,
},
]
)
with pytest.raises(
ValueError, match="Cannot format without a `config_path` argument."
):
parser.format(trial=trial)
def test_parse_from_args_only(
parser: OrionCmdlineParser,
commandline_fluff: List[str],
weird_argument: WeirdArgument,
):
"""Parse a commandline."""
cmd_args = commandline_fluff
parser.parse(cmd_args)
assert not parser.config_file_data
assert len(parser.cmd_priors) == 4
assert "/lr" in parser.cmd_priors
assert "/prior" in parser.cmd_priors
assert "/a.b" in parser.cmd_priors
assert f"/{weird_argument.name}" in parser.cmd_priors
assert parser.parser.template == [
"--seed",
"{seed}",
"--lr",
"{lr}",
"--non-prior",
"{non-prior}",
"--prior",
"{prior}",
"--a.b",
"{a.b}",
f"--{weird_argument.name}",
f"{{{weird_argument.name}}}",
"--some-path",
"{some-path}",
"--home-path",
"{home-path[0]}",
"{home-path[1]}",
]
class SpaceBuilder(object):
"""Build a `Space` object form user's configuration."""
def __init__(self):
"""Initialize a `SpaceBuilder`."""
self.dimbuilder = DimensionBuilder()
self.space = None
self.commands_tmpl = None
self.converter = None
self.parser = None
def build_from(self, config):
"""Build a `Space` object from a configuration.
Initialize a new parser for this commandline and parse the given config then
build a `Space` object from that configuration.
Returns
-------
`orion.algo.space.Space`
The problem's search space definition.
"""
self.parser = OrionCmdlineParser(orion_config.user_script_config)
self.parser.parse(config)
return self.build(self.parser.priors)
def build(self, configuration):
"""Create a definition of the problem's search space.
Using information from the user's script configuration (if provided) and the
command line arguments, will create a `Space` object defining the problem's
search space.
Parameters
----------
configuration: OrderedDict
An OrderedDict containing the name and the expression of the parameters.
Returns
-------
`orion.algo.space.Space`
The problem's search space definition.
"""
self.space = Space()
for namespace, expression in configuration.items():
if _should_not_be_built(expression):
continue
expression = _remove_marker(expression)
dimension = self.dimbuilder.build(namespace, expression)
try:
self.space.register(dimension)
except ValueError as exc:
error_msg = 'Conflict for name \'{}\' in parameters'.format(
namespace)
raise ValueError(error_msg) from exc
return self.space
def build_to(self, config_path, trial, experiment=None):
"""Create the configuration for the user's script.
Using the configuration parser, create the commandline associated with the
user's script while replacing the correct instances of parameter distributions by
their actual values. If needed, the parser will also create a configuration file.
Parameters
----------
config_path: str
Path in which the configuration file instance will be created.
trial: `orion.core.worker.trial.Trial`
Object with concrete parameter values for the defined `Space`.
experiment: `orion.core.worker.experiment.Experiment`, optional
Object with information related to the current experiment.
Returns
-------
list
The commandline arguments that must be given to script for execution.
"""
return self.parser.format(config_path, trial, experiment)
def test_infer_user_script(script_path):
"""Test that user script is infered correctly"""
parser = OrionCmdlineParser()
parser.parse(f"{script_path} and some args".split(" "))
assert parser.user_script == script_path
def test_infer_user_script_python(script_path: str):
"""Test that user script is infered correctly when using python"""
parser = OrionCmdlineParser()
parser.parse(f"python {script_path} and some args".split(" "))
assert parser.user_script == script_path
def test_format_commandline_and_config(
parser: OrionCmdlineParser,
commandline: List[str],
json_config: List[str],
tmp_path: Path,
json_converter,
weird_argument: WeirdArgument,
):
"""Format the commandline and a configuration file."""
cmd_args = json_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
trial = Trial(
params=[
{"name": "/lr", "type": "real", "value": -2.4},
{"name": "/prior", "type": "categorical", "value": "sgd"},
{"name": "/layers/1/width", "type": "integer", "value": 100},
{"name": "/layers/1/type", "type": "categorical", "value": "relu"},
{"name": "/layers/2/type", "type": "categorical", "value": "sigmoid"},
{"name": "/training/lr0", "type": "real", "value": 0.032},
{"name": "/training/mbs", "type": "integer", "value": 64},
{"name": "/something-same", "type": "categorical", "value": "3"},
{"name": "/a.b", "type": "real", "value": 0.3},
{
"name": f"/{weird_argument.name}",
"type": f"{weird_argument.prior_type}",
"value": weird_argument.value,
},
]
)
output_file = str(tmp_path / "output.json")
cmd_inst = parser.format(output_file, trial)
assert cmd_inst == [
"--config",
output_file,
"--seed",
"555",
"--lr",
"-2.4",
"--non-prior",
"choices({'sgd': 0.2, 'adam': 0.8})",
"--prior",
"sgd",
"--a.b",
"0.3",
f"--{weird_argument.name}",
f"{weird_argument.value}",
]
output_data = json_converter.parse(output_file)
assert output_data == {
"yo": 5,
"training": {"lr0": 0.032, "mbs": 64},
"layers": [
{"width": 64, "type": "relu"},
{"width": 100, "type": "relu"},
{"width": 16, "type": "sigmoid"},
],
"something-same": "3",
}