def fetch_metadata(user=None, user_args=None, user_script_config=None):
"""Infer rest information about the process + versioning"""
metadata = {"user": user if user else getpass.getuser()}
metadata["orion_version"] = orion.core.__version__
if user_args is None:
user_args = []
# Trailing white space are catched by argparse as an empty argument
if len(user_args) == 1 and user_args[0] == "":
user_args = []
if user_script_config is None:
user_script_config = orion.core.config.worker.user_script_config
cmdline_parser = OrionCmdlineParser(user_script_config)
cmdline_parser.parse(user_args)
if cmdline_parser.user_script:
# TODO: Remove this, it is all in cmdline_parser now
metadata["user_script"] = cmdline_parser.user_script
metadata["VCS"] = infer_versioning_metadata(cmdline_parser.user_script)
if user_args:
metadata["user_args"] = user_args
metadata["parser"] = cmdline_parser.get_state_dict()
metadata["user_script_config"] = user_script_config
metadata["priors"] = dict(cmdline_parser.priors)
return metadata
def build_from_config(self, config):
"""Build a fully configured (and writable) experiment based on full configuration.
.. seealso::
`orion.core.io.experiment_builder` for more information on the hierarchy of
configurations.
:class:`orion.core.worker.experiment.Experiment` for more information on the experiment
object.
"""
log.info(config)
# Pop out configuration concerning databases and resources
config.pop('database', None)
config.pop('resources', None)
experiment = Experiment(config['name'], config.get('user', None),
config.get('version', None))
# TODO: Handle both from cmdline and python APIs.
if 'priors' not in config['metadata'] and 'user_args' not in config['metadata']:
raise NoConfigurationError
# Parse to generate priors
if 'user_args' in config['metadata']:
parser = OrionCmdlineParser(orion.core.config.user_script_config)
parser.parse(config['metadata']['user_args'])
config['metadata']['parser'] = parser.get_state_dict()
config['metadata']['priors'] = dict(parser.priors)
# Finish experiment's configuration and write it to database.
experiment.configure(config)
return experiment
def populate_priors(metadata):
"""Compute parser state and priors based on user_args and populate metadata."""
if 'user_args' not in metadata:
return
parser = OrionCmdlineParser(orion.core.config.user_script_config)
parser.parse(metadata["user_args"])
metadata["parser"] = parser.get_state_dict()
metadata["priors"] = dict(parser.priors)
def populate_priors(metadata):
"""Compute parser state and priors based on user_args and populate metadata."""
if 'user_args' not in metadata:
return
update_user_args(metadata)
parser = OrionCmdlineParser(orion.core.config.worker.user_script_config,
allow_non_existing_user_script=True)
parser.parse(metadata["user_args"])
metadata["parser"] = parser.get_state_dict()
metadata["priors"] = dict(parser.priors)
def test_get_state_dict_before_parse(
parser: OrionCmdlineParser, commandline: List[str]
):
"""Test getting state dict."""
assert parser.get_state_dict() == {
"parser": {"keys": [], "arguments": [], "template": []},
"cmd_priors": list(map(list, parser.cmd_priors.items())),
"file_priors": list(map(list, parser.file_priors.items())),
"config_file_data": parser.config_file_data,
"config_prefix": parser.config_prefix,
"file_config_path": parser.file_config_path,
"converter": None,
}
def test_get_state_dict_after_parse_no_config_file(parser: OrionCmdlineParser,
commandline: List[str]):
"""Test getting state dict."""
parser.parse(commandline)
assert parser.get_state_dict() == {
"parser": parser.parser.get_state_dict(),
"cmd_priors": list(map(list, parser.cmd_priors.items())),
"file_priors": list(map(list, parser.file_priors.items())),
"config_file_data": parser.config_file_data,
"config_prefix": parser.config_prefix,
"file_config_path": parser.file_config_path,
"converter": None,
}
def test_get_state_dict_after_parse_no_config_file(commandline):
"""Test getting state dict."""
parser = OrionCmdlineParser()
parser.parse(commandline)
assert parser.get_state_dict() == {
'parser': parser.parser.get_state_dict(),
'cmd_priors': list(map(list, parser.cmd_priors.items())),
'file_priors': list(map(list, parser.file_priors.items())),
'config_file_data': parser.config_file_data,
'config_prefix': parser.config_prefix,
'file_config_path': parser.file_config_path,
'converter': None
}
def test_get_state_dict_before_parse(commandline):
"""Test getting state dict."""
parser = OrionCmdlineParser()
assert parser.get_state_dict() == {
'parser': {
'arguments': [],
'template': []
},
'cmd_priors': list(map(list, parser.cmd_priors.items())),
'file_priors': list(map(list, parser.file_priors.items())),
'config_file_data': parser.config_file_data,
'config_prefix': parser.config_prefix,
'file_config_path': parser.file_config_path,
'converter': None
}
def test_get_state_dict_after_parse_with_config_file(
parser: OrionCmdlineParser, yaml_config: List[str], commandline: List[str]
):
"""Test getting state dict."""
cmd_args = yaml_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
assert parser.get_state_dict() == {
"parser": parser.parser.get_state_dict(),
"cmd_priors": list(map(list, parser.cmd_priors.items())),
"file_priors": list(map(list, parser.file_priors.items())),
"config_file_data": parser.config_file_data,
"config_prefix": parser.config_prefix,
"file_config_path": parser.file_config_path,
"converter": parser.converter.get_state_dict(),
}
def test_get_state_dict_after_parse_with_config_file(yaml_config, commandline):
"""Test getting state dict."""
parser = OrionCmdlineParser()
cmd_args = yaml_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
assert parser.get_state_dict() == {
'parser': parser.parser.get_state_dict(),
'cmd_priors': list(map(list, parser.cmd_priors.items())),
'file_priors': list(map(list, parser.file_priors.items())),
'config_file_data': parser.config_file_data,
'config_prefix': parser.config_prefix,
'file_config_path': parser.file_config_path,
'converter': parser.converter.get_state_dict()
}
def update_dropout(experiment_config):
metadata = experiment_config["metadata"]
user_script = metadata.get("user_script", "")
user_args = metadata.get("user_args", [])
try:
index = user_args.index("--dropout")
except ValueError:
print(
f"No dropout for {experiment_config['metadata']}-v{experiment_config['version']}"
)
return
user_args[index + 1] = (user_args[index + 1].replace("5,", "0.5,").replace(
", discrete=True", ", precision=1"))
cmdline_parser = OrionCmdlineParser(allow_non_existing_files=True)
cmdline_parser.parse([user_script] + user_args)
metadata["parser"] = cmdline_parser.get_state_dict()
experiment_config["space"] = metadata["priors"] = dict(
cmdline_parser.priors)
# Update config in db
storage.update_experiment(uid=experiment_config["_id"],
**experiment_config)
# Update all trials in db (arf)
n_trials_before = len(storage.fetch_trials(uid=experiment_config["_id"]))
for trial in storage.fetch_trials(uid=experiment_config["_id"]):
previous_id = trial.id
for param in trial._params:
if param.name == "/dropout":
param.value /= 10
assert 0 <= param.value <= 0.5, param.value
storage.delete_trials(uid=experiment_config["_id"],
where=dict(_id=previous_id))
storage.register_trial(trial)
trials = storage.fetch_trials(uid=experiment_config["_id"])
assert len(trials) == n_trials_before, len(trials)
for trial in trials:
assert 0 <= trial.params["/dropout"] <= 0.5, trial
def populate_priors(metadata):
"""Compute parser state and priors based on user_args and populate metadata."""
if "user_args" not in metadata:
return
update_user_args(metadata)
parser = OrionCmdlineParser(orion.core.config.worker.user_script_config,
allow_non_existing_files=True)
if "parser" in metadata:
# To keep configs like config user_script_config
parser.config_prefix = metadata["parser"]["config_prefix"]
parser.parse(metadata["user_args"])
log.debug("Updating parser for backward compatibility")
metadata["parser"] = parser.get_state_dict()
log.debug(pprint.pformat(metadata["parser"]))
log.debug("Updating priors for backward compatibility")
metadata["priors"] = dict(parser.priors)
log.debug(pprint.pformat(metadata["priors"]))