def fetch_metadata(user=None, user_args=None, user_script_config=None): """Infer rest information about the process + versioning""" metadata = {"user": user if user else getpass.getuser()} metadata["orion_version"] = orion.core.__version__ if user_args is None: user_args = [] # Trailing white space are catched by argparse as an empty argument if len(user_args) == 1 and user_args[0] == "": user_args = [] if user_script_config is None: user_script_config = orion.core.config.worker.user_script_config cmdline_parser = OrionCmdlineParser(user_script_config) cmdline_parser.parse(user_args) if cmdline_parser.user_script: # TODO: Remove this, it is all in cmdline_parser now metadata["user_script"] = cmdline_parser.user_script metadata["VCS"] = infer_versioning_metadata(cmdline_parser.user_script) if user_args: metadata["user_args"] = user_args metadata["parser"] = cmdline_parser.get_state_dict() metadata["user_script_config"] = user_script_config metadata["priors"] = dict(cmdline_parser.priors) return metadata
def build_from_config(self, config): """Build a fully configured (and writable) experiment based on full configuration. .. seealso:: `orion.core.io.experiment_builder` for more information on the hierarchy of configurations. :class:`orion.core.worker.experiment.Experiment` for more information on the experiment object. """ log.info(config) # Pop out configuration concerning databases and resources config.pop('database', None) config.pop('resources', None) experiment = Experiment(config['name'], config.get('user', None), config.get('version', None)) # TODO: Handle both from cmdline and python APIs. if 'priors' not in config['metadata'] and 'user_args' not in config['metadata']: raise NoConfigurationError # Parse to generate priors if 'user_args' in config['metadata']: parser = OrionCmdlineParser(orion.core.config.user_script_config) parser.parse(config['metadata']['user_args']) config['metadata']['parser'] = parser.get_state_dict() config['metadata']['priors'] = dict(parser.priors) # Finish experiment's configuration and write it to database. experiment.configure(config) return experiment
def test_format_commandline_only( parser: OrionCmdlineParser, commandline: List[str], weird_argument: WeirdArgument ): """Format the commandline using only args.""" parser.parse(commandline) trial = Trial( params=[ {"name": "/lr", "type": "real", "value": -2.4}, {"name": "/prior", "type": "categorical", "value": "sgd"}, {"name": "/a.b", "type": "real", "value": 0.5}, { "name": f"/{weird_argument.name}", "type": weird_argument.prior_type, "value": weird_argument.value, }, ] ) cmd_inst = parser.format(trial=trial) assert cmd_inst == [ "--seed", "555", "--lr", "-2.4", "--non-prior", "choices({'sgd': 0.2, 'adam': 0.8})", "--prior", "sgd", "--a.b", "0.5", f"--{weird_argument.name}", f"{weird_argument.value}", ]
def fetch_metadata(user=None, user_args=None): """Infer rest information about the process + versioning""" metadata = {'user': user if user else getpass.getuser()} metadata['orion_version'] = orion.core.__version__ if user_args is None: user_args = [] # Trailing white space are catched by argparse as an empty argument if len(user_args) == 1 and user_args[0] == '': user_args = [] cmdline_parser = OrionCmdlineParser(config.worker.user_script_config) cmdline_parser.parse(user_args) if cmdline_parser.user_script: # TODO: Remove this, it is all in cmdline_parser now metadata['user_script'] = cmdline_parser.user_script metadata['VCS'] = infer_versioning_metadata(cmdline_parser.user_script) if user_args: # TODO: Remove this, it is all in cmdline_parser now metadata['user_args'] = user_args return metadata
def test_format_with_properties( parser: OrionCmdlineParser, cmd_with_properties: List[str], hacked_exp: Experiment, weird_argument: WeirdArgument, ): """Test if format correctly puts the value of `trial` and `exp` when used as properties""" parser.parse(cmd_with_properties) # NOTE: Also using a weird argument here, to make sure the parser is able to distinguish # property look-up vs weird argument names. trial = Trial( experiment="trial_test", params=[ {"name": "/lr", "type": "real", "value": -2.4}, {"name": "/prior", "type": "categorical", "value": "sgd"}, { "name": f"/{weird_argument.name}", "type": weird_argument.prior_type, "value": weird_argument.value, }, ], ) cmd_line = parser.format(None, trial=trial, experiment=hacked_exp) assert trial.hash_name in cmd_line assert "supernaedo2-dendi" in cmd_line
def populate_priors(metadata): """Compute parser state and priors based on user_args and populate metadata.""" if 'user_args' not in metadata: return parser = OrionCmdlineParser(orion.core.config.user_script_config) parser.parse(metadata["user_args"]) metadata["parser"] = parser.get_state_dict() metadata["priors"] = dict(parser.priors)
def test_parse_equivalency(yaml_config: List[str], json_config: List[str]): """Templates found from json and yaml are the same.""" parser_yaml = OrionCmdlineParser(allow_non_existing_files=True) parser_yaml.parse(yaml_config) dict_from_yaml = parser_yaml.config_file_data parser_json = OrionCmdlineParser(allow_non_existing_files=True) parser_json.parse(json_config) dict_from_json = parser_json.config_file_data assert dict_from_json == dict_from_yaml
def test_configurable_config_arg_do_not_exist(script_path: str): """Test that parser can handle command if config file does not exist""" parser = OrionCmdlineParser() command = f"python {script_path} --config idontexist.yaml".split(" ") with pytest.raises(OSError) as exc: parser.parse(command) assert exc.match("The path specified for the script config does not exist") parser = OrionCmdlineParser(allow_non_existing_files=True) parser.parse(command)
def test_parse_equivalency(yaml_config, json_config): """Templates found from json and yaml are the same.""" parser_yaml = OrionCmdlineParser() parser_yaml.parse(yaml_config) dict_from_yaml = parser_yaml.config_file_data parser_json = OrionCmdlineParser() parser_json.parse(json_config) dict_from_json = parser_json.config_file_data assert dict_from_json == dict_from_yaml
def test_parse_finds_conflict(parser: OrionCmdlineParser, commandline: List[str], yaml_config: List[str]): """Parse find conflicting declaration in commandline and config file.""" cmd_args = yaml_config cmd_args.extend(commandline) cmd_args.extend(["--something-same~choices({'sgd': 0.2, 'adam': 0.8})"]) with pytest.raises(ValueError) as exc: parser.parse(cmd_args) assert "Conflict" in str(exc.value)
def test_infer_user_script_when_missing(): """Test that user script is infered correctly even if missing""" parser = OrionCmdlineParser() with pytest.raises(FileNotFoundError) as exc: parser.parse("python script.py and some args".split(" ")) assert exc.match("The path specified for the script does not exist") parser = OrionCmdlineParser(allow_non_existing_files=True) parser.parse("python script.py and some args".split(" ")) assert parser.user_script == "script.py"
def populate_priors(metadata): """Compute parser state and priors based on user_args and populate metadata.""" if 'user_args' not in metadata: return update_user_args(metadata) parser = OrionCmdlineParser(orion.core.config.worker.user_script_config, allow_non_existing_user_script=True) parser.parse(metadata["user_args"]) metadata["parser"] = parser.get_state_dict() metadata["priors"] = dict(parser.priors)
def test_parse_from_json_config(parser: OrionCmdlineParser, json_config: List[str]): """Parse from a json config only.""" parser.parse(json_config) config = parser.priors assert len(config.keys()) == 6 assert "/layers/1/width" in config assert "/layers/1/type" in config assert "/layers/2/type" in config assert "/training/lr0" in config assert "/training/mbs" in config assert "/something-same" in config
def test_configurable_config_arg(parser_diff_prefix: OrionCmdlineParser, yaml_sample_path: str): """Parse from a yaml config only.""" parser_diff_prefix.parse(["--config2", yaml_sample_path]) config = parser_diff_prefix.priors assert len(config.keys()) == 6 assert "/layers/1/width" in config assert "/layers/1/type" in config assert "/layers/2/type" in config assert "/training/lr0" in config assert "/training/mbs" in config assert "/something-same" in config
def test_get_state_dict_after_parse_no_config_file(parser: OrionCmdlineParser, commandline: List[str]): """Test getting state dict.""" parser.parse(commandline) assert parser.get_state_dict() == { "parser": parser.parser.get_state_dict(), "cmd_priors": list(map(list, parser.cmd_priors.items())), "file_priors": list(map(list, parser.file_priors.items())), "config_file_data": parser.config_file_data, "config_prefix": parser.config_prefix, "file_config_path": parser.file_config_path, "converter": None, }
def test_get_state_dict_after_parse_no_config_file(commandline): """Test getting state dict.""" parser = OrionCmdlineParser() parser.parse(commandline) assert parser.get_state_dict() == { 'parser': parser.parser.get_state_dict(), 'cmd_priors': list(map(list, parser.cmd_priors.items())), 'file_priors': list(map(list, parser.file_priors.items())), 'config_file_data': parser.config_file_data, 'config_prefix': parser.config_prefix, 'file_config_path': parser.file_config_path, 'converter': None }
def test_get_state_dict_after_parse_with_config_file( parser: OrionCmdlineParser, yaml_config: List[str], commandline: List[str] ): """Test getting state dict.""" cmd_args = yaml_config cmd_args.extend(commandline) parser.parse(cmd_args) assert parser.get_state_dict() == { "parser": parser.parser.get_state_dict(), "cmd_priors": list(map(list, parser.cmd_priors.items())), "file_priors": list(map(list, parser.file_priors.items())), "config_file_data": parser.config_file_data, "config_prefix": parser.config_prefix, "file_config_path": parser.file_config_path, "converter": parser.converter.get_state_dict(), }
def test_get_state_dict_after_parse_with_config_file(yaml_config, commandline): """Test getting state dict.""" parser = OrionCmdlineParser() cmd_args = yaml_config cmd_args.extend(commandline) parser.parse(cmd_args) assert parser.get_state_dict() == { 'parser': parser.parser.get_state_dict(), 'cmd_priors': list(map(list, parser.cmd_priors.items())), 'file_priors': list(map(list, parser.file_priors.items())), 'config_file_data': parser.config_file_data, 'config_prefix': parser.config_prefix, 'file_config_path': parser.file_config_path, 'converter': parser.converter.get_state_dict() }
def test_parse_from_args_and_config_yaml( parser: OrionCmdlineParser, commandline: List[str], yaml_config: List[str], weird_argument: WeirdArgument, ): """Parse both from commandline and config file.""" cmd_args = yaml_config cmd_args.extend(commandline) parser.parse(cmd_args) config = parser.priors assert len(config) == 10 assert "/lr" in config assert "/prior" in config assert "/layers/1/width" in config assert "/layers/1/type" in config assert "/layers/2/type" in config assert "/training/lr0" in config assert "/training/mbs" in config assert "/something-same" in config assert "/a.b" in config assert f"/{weird_argument.name}" in config template = parser.parser.template assert template == [ "--config", "{config}", "--seed", "{seed}", "--lr", "{lr}", "--non-prior", "{non-prior}", "--prior", "{prior}", "--a.b", "{a.b}", f"--{weird_argument.name}", f"{{{weird_argument.name}}}", ]
def update_dropout(experiment_config): metadata = experiment_config["metadata"] user_script = metadata.get("user_script", "") user_args = metadata.get("user_args", []) try: index = user_args.index("--dropout") except ValueError: print( f"No dropout for {experiment_config['metadata']}-v{experiment_config['version']}" ) return user_args[index + 1] = (user_args[index + 1].replace("5,", "0.5,").replace( ", discrete=True", ", precision=1")) cmdline_parser = OrionCmdlineParser(allow_non_existing_files=True) cmdline_parser.parse([user_script] + user_args) metadata["parser"] = cmdline_parser.get_state_dict() experiment_config["space"] = metadata["priors"] = dict( cmdline_parser.priors) # Update config in db storage.update_experiment(uid=experiment_config["_id"], **experiment_config) # Update all trials in db (arf) n_trials_before = len(storage.fetch_trials(uid=experiment_config["_id"])) for trial in storage.fetch_trials(uid=experiment_config["_id"]): previous_id = trial.id for param in trial._params: if param.name == "/dropout": param.value /= 10 assert 0 <= param.value <= 0.5, param.value storage.delete_trials(uid=experiment_config["_id"], where=dict(_id=previous_id)) storage.register_trial(trial) trials = storage.fetch_trials(uid=experiment_config["_id"]) assert len(trials) == n_trials_before, len(trials) for trial in trials: assert 0 <= trial.params["/dropout"] <= 0.5, trial
def populate_priors(metadata): """Compute parser state and priors based on user_args and populate metadata.""" if "user_args" not in metadata: return update_user_args(metadata) parser = OrionCmdlineParser(orion.core.config.worker.user_script_config, allow_non_existing_files=True) if "parser" in metadata: # To keep configs like config user_script_config parser.config_prefix = metadata["parser"]["config_prefix"] parser.parse(metadata["user_args"]) log.debug("Updating parser for backward compatibility") metadata["parser"] = parser.get_state_dict() log.debug(pprint.pformat(metadata["parser"])) log.debug("Updating priors for backward compatibility") metadata["priors"] = dict(parser.priors) log.debug(pprint.pformat(metadata["priors"]))
def test_format_without_config_path( parser: OrionCmdlineParser, commandline: List[str], json_config: List[str], tmp_path: Path, json_converter: JSONConverter, weird_argument: WeirdArgument, ): """Verify that parser.format() raises ValueError when config path not passed.""" cmd_args = json_config cmd_args.extend(commandline) parser.parse(cmd_args) trial = Trial( params=[ {"name": "/lr", "type": "real", "value": -2.4}, {"name": "/prior", "type": "categorical", "value": "sgd"}, {"name": "/layers/1/width", "type": "integer", "value": 100}, {"name": "/layers/1/type", "type": "categorical", "value": "relu"}, {"name": "/layers/2/type", "type": "categorical", "value": "sigmoid"}, {"name": "/training/lr0", "type": "real", "value": 0.032}, {"name": "/training/mbs", "type": "integer", "value": 64}, {"name": "/something-same", "type": "categorical", "value": "3"}, { "name": f"/{weird_argument.name}", "type": "categorical", "value": weird_argument.value, }, ] ) with pytest.raises( ValueError, match="Cannot format without a `config_path` argument." ): parser.format(trial=trial)
def test_parse_from_args_only( parser: OrionCmdlineParser, commandline_fluff: List[str], weird_argument: WeirdArgument, ): """Parse a commandline.""" cmd_args = commandline_fluff parser.parse(cmd_args) assert not parser.config_file_data assert len(parser.cmd_priors) == 4 assert "/lr" in parser.cmd_priors assert "/prior" in parser.cmd_priors assert "/a.b" in parser.cmd_priors assert f"/{weird_argument.name}" in parser.cmd_priors assert parser.parser.template == [ "--seed", "{seed}", "--lr", "{lr}", "--non-prior", "{non-prior}", "--prior", "{prior}", "--a.b", "{a.b}", f"--{weird_argument.name}", f"{{{weird_argument.name}}}", "--some-path", "{some-path}", "--home-path", "{home-path[0]}", "{home-path[1]}", ]
class SpaceBuilder(object): """Build a `Space` object form user's configuration.""" def __init__(self): """Initialize a `SpaceBuilder`.""" self.dimbuilder = DimensionBuilder() self.space = None self.commands_tmpl = None self.converter = None self.parser = None def build_from(self, config): """Build a `Space` object from a configuration. Initialize a new parser for this commandline and parse the given config then build a `Space` object from that configuration. Returns ------- `orion.algo.space.Space` The problem's search space definition. """ self.parser = OrionCmdlineParser(orion_config.user_script_config) self.parser.parse(config) return self.build(self.parser.priors) def build(self, configuration): """Create a definition of the problem's search space. Using information from the user's script configuration (if provided) and the command line arguments, will create a `Space` object defining the problem's search space. Parameters ---------- configuration: OrderedDict An OrderedDict containing the name and the expression of the parameters. Returns ------- `orion.algo.space.Space` The problem's search space definition. """ self.space = Space() for namespace, expression in configuration.items(): if _should_not_be_built(expression): continue expression = _remove_marker(expression) dimension = self.dimbuilder.build(namespace, expression) try: self.space.register(dimension) except ValueError as exc: error_msg = 'Conflict for name \'{}\' in parameters'.format( namespace) raise ValueError(error_msg) from exc return self.space def build_to(self, config_path, trial, experiment=None): """Create the configuration for the user's script. Using the configuration parser, create the commandline associated with the user's script while replacing the correct instances of parameter distributions by their actual values. If needed, the parser will also create a configuration file. Parameters ---------- config_path: str Path in which the configuration file instance will be created. trial: `orion.core.worker.trial.Trial` Object with concrete parameter values for the defined `Space`. experiment: `orion.core.worker.experiment.Experiment`, optional Object with information related to the current experiment. Returns ------- list The commandline arguments that must be given to script for execution. """ return self.parser.format(config_path, trial, experiment)
def test_infer_user_script(script_path): """Test that user script is infered correctly""" parser = OrionCmdlineParser() parser.parse(f"{script_path} and some args".split(" ")) assert parser.user_script == script_path
def test_infer_user_script_python(script_path: str): """Test that user script is infered correctly when using python""" parser = OrionCmdlineParser() parser.parse(f"python {script_path} and some args".split(" ")) assert parser.user_script == script_path
def test_format_commandline_and_config( parser: OrionCmdlineParser, commandline: List[str], json_config: List[str], tmp_path: Path, json_converter, weird_argument: WeirdArgument, ): """Format the commandline and a configuration file.""" cmd_args = json_config cmd_args.extend(commandline) parser.parse(cmd_args) trial = Trial( params=[ {"name": "/lr", "type": "real", "value": -2.4}, {"name": "/prior", "type": "categorical", "value": "sgd"}, {"name": "/layers/1/width", "type": "integer", "value": 100}, {"name": "/layers/1/type", "type": "categorical", "value": "relu"}, {"name": "/layers/2/type", "type": "categorical", "value": "sigmoid"}, {"name": "/training/lr0", "type": "real", "value": 0.032}, {"name": "/training/mbs", "type": "integer", "value": 64}, {"name": "/something-same", "type": "categorical", "value": "3"}, {"name": "/a.b", "type": "real", "value": 0.3}, { "name": f"/{weird_argument.name}", "type": f"{weird_argument.prior_type}", "value": weird_argument.value, }, ] ) output_file = str(tmp_path / "output.json") cmd_inst = parser.format(output_file, trial) assert cmd_inst == [ "--config", output_file, "--seed", "555", "--lr", "-2.4", "--non-prior", "choices({'sgd': 0.2, 'adam': 0.8})", "--prior", "sgd", "--a.b", "0.3", f"--{weird_argument.name}", f"{weird_argument.value}", ] output_data = json_converter.parse(output_file) assert output_data == { "yo": 5, "training": {"lr0": 0.032, "mbs": 64}, "layers": [ {"width": 64, "type": "relu"}, {"width": 100, "type": "relu"}, {"width": 16, "type": "sigmoid"}, ], "something-same": "3", }