def test_format_commandline_only( parser: OrionCmdlineParser, commandline: List[str], weird_argument: WeirdArgument ): """Format the commandline using only args.""" parser.parse(commandline) trial = Trial( params=[ {"name": "/lr", "type": "real", "value": -2.4}, {"name": "/prior", "type": "categorical", "value": "sgd"}, {"name": "/a.b", "type": "real", "value": 0.5}, { "name": f"/{weird_argument.name}", "type": weird_argument.prior_type, "value": weird_argument.value, }, ] ) cmd_inst = parser.format(trial=trial) assert cmd_inst == [ "--seed", "555", "--lr", "-2.4", "--non-prior", "choices({'sgd': 0.2, 'adam': 0.8})", "--prior", "sgd", "--a.b", "0.5", f"--{weird_argument.name}", f"{weird_argument.value}", ]
def __init__( self, experiment, user_script_config=None, interrupt_signal_code=None, ignore_code_changes=None, ): log.debug("Creating Consumer object.") self.experiment = experiment self.space = experiment.space if self.space is None: raise RuntimeError( "Experiment object provided to Consumer has not yet completed" " initialization." ) if user_script_config is None: user_script_config = orion.core.config.worker.user_script_config if interrupt_signal_code is None: interrupt_signal_code = orion.core.config.worker.interrupt_signal_code # NOTE: If ignore_code_changes is None, we can assume EVC is enabled. if ignore_code_changes is None: ignore_code_changes = orion.core.config.evc.ignore_code_changes self.interrupt_signal_code = interrupt_signal_code self.ignore_code_changes = ignore_code_changes # Fetch space builder self.template_builder = OrionCmdlineParser(user_script_config) self.template_builder.set_state_dict(experiment.metadata["parser"]) self.pacemaker = None
def __init__(self, experiment): """Initialize a consumer. :param experiment: Manager of this experiment, provides convenient interface for interacting with the database. """ log.debug("Creating Consumer object.") self.experiment = experiment self.space = experiment.space if self.space is None: raise RuntimeError( "Experiment object provided to Consumer has not yet completed" " initialization.") # Fetch space builder self.template_builder = OrionCmdlineParser( orion.core.config.user_script_config) self.template_builder.set_state_dict(experiment.metadata['parser']) # Get path to user's script and infer trial configuration directory if experiment.working_dir: self.working_dir = os.path.abspath(experiment.working_dir) else: self.working_dir = os.path.join(tempfile.gettempdir(), 'orion') self.script_path = experiment.metadata['user_script'] self.pacemaker = None
def test_format_with_properties( parser: OrionCmdlineParser, cmd_with_properties: List[str], hacked_exp: Experiment, weird_argument: WeirdArgument, ): """Test if format correctly puts the value of `trial` and `exp` when used as properties""" parser.parse(cmd_with_properties) # NOTE: Also using a weird argument here, to make sure the parser is able to distinguish # property look-up vs weird argument names. trial = Trial( experiment="trial_test", params=[ {"name": "/lr", "type": "real", "value": -2.4}, {"name": "/prior", "type": "categorical", "value": "sgd"}, { "name": f"/{weird_argument.name}", "type": weird_argument.prior_type, "value": weird_argument.value, }, ], ) cmd_line = parser.format(None, trial=trial, experiment=hacked_exp) assert trial.hash_name in cmd_line assert "supernaedo2-dendi" in cmd_line
def fetch_metadata(user=None, user_args=None, user_script_config=None): """Infer rest information about the process + versioning""" metadata = {"user": user if user else getpass.getuser()} metadata["orion_version"] = orion.core.__version__ if user_args is None: user_args = [] # Trailing white space are catched by argparse as an empty argument if len(user_args) == 1 and user_args[0] == "": user_args = [] if user_script_config is None: user_script_config = orion.core.config.worker.user_script_config cmdline_parser = OrionCmdlineParser(user_script_config) cmdline_parser.parse(user_args) if cmdline_parser.user_script: # TODO: Remove this, it is all in cmdline_parser now metadata["user_script"] = cmdline_parser.user_script metadata["VCS"] = infer_versioning_metadata(cmdline_parser.user_script) if user_args: metadata["user_args"] = user_args metadata["parser"] = cmdline_parser.get_state_dict() metadata["user_script_config"] = user_script_config metadata["priors"] = dict(cmdline_parser.priors) return metadata
def build_from_config(self, config): """Build a fully configured (and writable) experiment based on full configuration. .. seealso:: `orion.core.io.experiment_builder` for more information on the hierarchy of configurations. :class:`orion.core.worker.experiment.Experiment` for more information on the experiment object. """ log.info(config) # Pop out configuration concerning databases and resources config.pop('database', None) config.pop('resources', None) experiment = Experiment(config['name'], config.get('user', None), config.get('version', None)) # TODO: Handle both from cmdline and python APIs. if 'priors' not in config['metadata'] and 'user_args' not in config['metadata']: raise NoConfigurationError # Parse to generate priors if 'user_args' in config['metadata']: parser = OrionCmdlineParser(orion.core.config.user_script_config) parser.parse(config['metadata']['user_args']) config['metadata']['parser'] = parser.get_state_dict() config['metadata']['priors'] = dict(parser.priors) # Finish experiment's configuration and write it to database. experiment.configure(config) return experiment
def fetch_metadata(user=None, user_args=None): """Infer rest information about the process + versioning""" metadata = {'user': user if user else getpass.getuser()} metadata['orion_version'] = orion.core.__version__ if user_args is None: user_args = [] # Trailing white space are catched by argparse as an empty argument if len(user_args) == 1 and user_args[0] == '': user_args = [] cmdline_parser = OrionCmdlineParser(config.worker.user_script_config) cmdline_parser.parse(user_args) if cmdline_parser.user_script: # TODO: Remove this, it is all in cmdline_parser now metadata['user_script'] = cmdline_parser.user_script metadata['VCS'] = infer_versioning_metadata(cmdline_parser.user_script) if user_args: # TODO: Remove this, it is all in cmdline_parser now metadata['user_args'] = user_args return metadata
def __init__(self, experiment, heartbeat=None, user_script_config=None, interrupt_signal_code=None): log.debug("Creating Consumer object.") self.experiment = experiment self.space = experiment.space if self.space is None: raise RuntimeError("Experiment object provided to Consumer has not yet completed" " initialization.") if heartbeat is None: heartbeat = orion.core.config.worker.heartbeat if user_script_config is None: user_script_config = orion.core.config.worker.user_script_config if interrupt_signal_code is None: interrupt_signal_code = orion.core.config.worker.interrupt_signal_code self.heartbeat = heartbeat self.interrupt_signal_code = interrupt_signal_code # Fetch space builder self.template_builder = OrionCmdlineParser(user_script_config) self.template_builder.set_state_dict(experiment.metadata['parser']) # Get path to user's script and infer trial configuration directory if experiment.working_dir: self.working_dir = os.path.abspath(experiment.working_dir) else: self.working_dir = os.path.join(tempfile.gettempdir(), 'orion') self.pacemaker = None
def populate_priors(metadata): """Compute parser state and priors based on user_args and populate metadata.""" if 'user_args' not in metadata: return parser = OrionCmdlineParser(orion.core.config.user_script_config) parser.parse(metadata["user_args"]) metadata["parser"] = parser.get_state_dict() metadata["priors"] = dict(parser.priors)
def test_parse_finds_conflict(parser: OrionCmdlineParser, commandline: List[str], yaml_config: List[str]): """Parse find conflicting declaration in commandline and config file.""" cmd_args = yaml_config cmd_args.extend(commandline) cmd_args.extend(["--something-same~choices({'sgd': 0.2, 'adam': 0.8})"]) with pytest.raises(ValueError) as exc: parser.parse(cmd_args) assert "Conflict" in str(exc.value)
def populate_priors(metadata): """Compute parser state and priors based on user_args and populate metadata.""" if 'user_args' not in metadata: return update_user_args(metadata) parser = OrionCmdlineParser(orion.core.config.worker.user_script_config, allow_non_existing_user_script=True) parser.parse(metadata["user_args"]) metadata["parser"] = parser.get_state_dict() metadata["priors"] = dict(parser.priors)
def test_parse_from_json_config(parser: OrionCmdlineParser, json_config: List[str]): """Parse from a json config only.""" parser.parse(json_config) config = parser.priors assert len(config.keys()) == 6 assert "/layers/1/width" in config assert "/layers/1/type" in config assert "/layers/2/type" in config assert "/training/lr0" in config assert "/training/mbs" in config assert "/something-same" in config
def test_configurable_config_arg(parser_diff_prefix: OrionCmdlineParser, yaml_sample_path: str): """Parse from a yaml config only.""" parser_diff_prefix.parse(["--config2", yaml_sample_path]) config = parser_diff_prefix.priors assert len(config.keys()) == 6 assert "/layers/1/width" in config assert "/layers/1/type" in config assert "/layers/2/type" in config assert "/training/lr0" in config assert "/training/mbs" in config assert "/something-same" in config
def test_get_state_dict_after_parse_no_config_file(parser: OrionCmdlineParser, commandline: List[str]): """Test getting state dict.""" parser.parse(commandline) assert parser.get_state_dict() == { "parser": parser.parser.get_state_dict(), "cmd_priors": list(map(list, parser.cmd_priors.items())), "file_priors": list(map(list, parser.file_priors.items())), "config_file_data": parser.config_file_data, "config_prefix": parser.config_prefix, "file_config_path": parser.file_config_path, "converter": None, }
def test_get_state_dict_after_parse_no_config_file(commandline): """Test getting state dict.""" parser = OrionCmdlineParser() parser.parse(commandline) assert parser.get_state_dict() == { 'parser': parser.parser.get_state_dict(), 'cmd_priors': list(map(list, parser.cmd_priors.items())), 'file_priors': list(map(list, parser.file_priors.items())), 'config_file_data': parser.config_file_data, 'config_prefix': parser.config_prefix, 'file_config_path': parser.file_config_path, 'converter': None }
def build_from(self, config): """Build a `Space` object from a configuration. Initialize a new parser for this commandline and parse the given config then build a `Space` object from that configuration. Returns ------- `orion.algo.space.Space` The problem's search space definition. """ self.parser = OrionCmdlineParser(orion_config.user_script_config) self.parser.parse(config) return self.build(self.parser.priors)
def test_get_state_dict_before_parse(commandline): """Test getting state dict.""" parser = OrionCmdlineParser() assert parser.get_state_dict() == { 'parser': { 'arguments': [], 'template': [] }, 'cmd_priors': list(map(list, parser.cmd_priors.items())), 'file_priors': list(map(list, parser.file_priors.items())), 'config_file_data': parser.config_file_data, 'config_prefix': parser.config_prefix, 'file_config_path': parser.file_config_path, 'converter': None }
def update_metadata(metadata): """Update information about the process + versioning""" metadata.setdefault("user", getpass.getuser()) metadata["orion_version"] = orion.core.__version__ if not metadata.get("user_args"): return metadata cmdline_parser = OrionCmdlineParser() cmdline_parser.set_state_dict(metadata["parser"]) if cmdline_parser.user_script: # TODO: Remove this, it is all in cmdline_parser now metadata["user_script"] = cmdline_parser.user_script metadata["VCS"] = infer_versioning_metadata(cmdline_parser.user_script) return metadata
def test_parse_equivalency(yaml_config, json_config): """Templates found from json and yaml are the same.""" parser_yaml = OrionCmdlineParser() parser_yaml.parse(yaml_config) dict_from_yaml = parser_yaml.config_file_data parser_json = OrionCmdlineParser() parser_json.parse(json_config) dict_from_json = parser_json.config_file_data assert dict_from_json == dict_from_yaml
def test_configurable_config_arg_do_not_exist(script_path: str): """Test that parser can handle command if config file does not exist""" parser = OrionCmdlineParser() command = f"python {script_path} --config idontexist.yaml".split(" ") with pytest.raises(OSError) as exc: parser.parse(command) assert exc.match("The path specified for the script config does not exist") parser = OrionCmdlineParser(allow_non_existing_files=True) parser.parse(command)
def test_parse_equivalency(yaml_config: List[str], json_config: List[str]): """Templates found from json and yaml are the same.""" parser_yaml = OrionCmdlineParser(allow_non_existing_files=True) parser_yaml.parse(yaml_config) dict_from_yaml = parser_yaml.config_file_data parser_json = OrionCmdlineParser(allow_non_existing_files=True) parser_json.parse(json_config) dict_from_json = parser_json.config_file_data assert dict_from_json == dict_from_yaml
def test_get_state_dict_after_parse_with_config_file( parser: OrionCmdlineParser, yaml_config: List[str], commandline: List[str] ): """Test getting state dict.""" cmd_args = yaml_config cmd_args.extend(commandline) parser.parse(cmd_args) assert parser.get_state_dict() == { "parser": parser.parser.get_state_dict(), "cmd_priors": list(map(list, parser.cmd_priors.items())), "file_priors": list(map(list, parser.file_priors.items())), "config_file_data": parser.config_file_data, "config_prefix": parser.config_prefix, "file_config_path": parser.file_config_path, "converter": parser.converter.get_state_dict(), }
def test_get_state_dict_after_parse_with_config_file(yaml_config, commandline): """Test getting state dict.""" parser = OrionCmdlineParser() cmd_args = yaml_config cmd_args.extend(commandline) parser.parse(cmd_args) assert parser.get_state_dict() == { 'parser': parser.parser.get_state_dict(), 'cmd_priors': list(map(list, parser.cmd_priors.items())), 'file_priors': list(map(list, parser.file_priors.items())), 'config_file_data': parser.config_file_data, 'config_prefix': parser.config_prefix, 'file_config_path': parser.file_config_path, 'converter': parser.converter.get_state_dict() }
def test_infer_user_script_when_missing(): """Test that user script is infered correctly even if missing""" parser = OrionCmdlineParser() with pytest.raises(FileNotFoundError) as exc: parser.parse("python script.py and some args".split(" ")) assert exc.match("The path specified for the script does not exist") parser = OrionCmdlineParser(allow_non_existing_files=True) parser.parse("python script.py and some args".split(" ")) assert parser.user_script == "script.py"
def test_parse_from_args_and_config_yaml( parser: OrionCmdlineParser, commandline: List[str], yaml_config: List[str], weird_argument: WeirdArgument, ): """Parse both from commandline and config file.""" cmd_args = yaml_config cmd_args.extend(commandline) parser.parse(cmd_args) config = parser.priors assert len(config) == 10 assert "/lr" in config assert "/prior" in config assert "/layers/1/width" in config assert "/layers/1/type" in config assert "/layers/2/type" in config assert "/training/lr0" in config assert "/training/mbs" in config assert "/something-same" in config assert "/a.b" in config assert f"/{weird_argument.name}" in config template = parser.parser.template assert template == [ "--config", "{config}", "--seed", "{seed}", "--lr", "{lr}", "--non-prior", "{non-prior}", "--prior", "{prior}", "--a.b", "{a.b}", f"--{weird_argument.name}", f"{{{weird_argument.name}}}", ]
def update_dropout(experiment_config): metadata = experiment_config["metadata"] user_script = metadata.get("user_script", "") user_args = metadata.get("user_args", []) try: index = user_args.index("--dropout") except ValueError: print( f"No dropout for {experiment_config['metadata']}-v{experiment_config['version']}" ) return user_args[index + 1] = (user_args[index + 1].replace("5,", "0.5,").replace( ", discrete=True", ", precision=1")) cmdline_parser = OrionCmdlineParser(allow_non_existing_files=True) cmdline_parser.parse([user_script] + user_args) metadata["parser"] = cmdline_parser.get_state_dict() experiment_config["space"] = metadata["priors"] = dict( cmdline_parser.priors) # Update config in db storage.update_experiment(uid=experiment_config["_id"], **experiment_config) # Update all trials in db (arf) n_trials_before = len(storage.fetch_trials(uid=experiment_config["_id"])) for trial in storage.fetch_trials(uid=experiment_config["_id"]): previous_id = trial.id for param in trial._params: if param.name == "/dropout": param.value /= 10 assert 0 <= param.value <= 0.5, param.value storage.delete_trials(uid=experiment_config["_id"], where=dict(_id=previous_id)) storage.register_trial(trial) trials = storage.fetch_trials(uid=experiment_config["_id"]) assert len(trials) == n_trials_before, len(trials) for trial in trials: assert 0 <= trial.params["/dropout"] <= 0.5, trial
def test_get_state_dict_before_parse( parser: OrionCmdlineParser, commandline: List[str] ): """Test getting state dict.""" assert parser.get_state_dict() == { "parser": {"keys": [], "arguments": [], "template": []}, "cmd_priors": list(map(list, parser.cmd_priors.items())), "file_priors": list(map(list, parser.file_priors.items())), "config_file_data": parser.config_file_data, "config_prefix": parser.config_prefix, "file_config_path": parser.file_config_path, "converter": None, }
def test_format_without_config_path( parser: OrionCmdlineParser, commandline: List[str], json_config: List[str], tmp_path: Path, json_converter: JSONConverter, weird_argument: WeirdArgument, ): """Verify that parser.format() raises ValueError when config path not passed.""" cmd_args = json_config cmd_args.extend(commandline) parser.parse(cmd_args) trial = Trial( params=[ {"name": "/lr", "type": "real", "value": -2.4}, {"name": "/prior", "type": "categorical", "value": "sgd"}, {"name": "/layers/1/width", "type": "integer", "value": 100}, {"name": "/layers/1/type", "type": "categorical", "value": "relu"}, {"name": "/layers/2/type", "type": "categorical", "value": "sigmoid"}, {"name": "/training/lr0", "type": "real", "value": 0.032}, {"name": "/training/mbs", "type": "integer", "value": 64}, {"name": "/something-same", "type": "categorical", "value": "3"}, { "name": f"/{weird_argument.name}", "type": "categorical", "value": weird_argument.value, }, ] ) with pytest.raises( ValueError, match="Cannot format without a `config_path` argument." ): parser.format(trial=trial)
def test_parse_from_args_only( parser: OrionCmdlineParser, commandline_fluff: List[str], weird_argument: WeirdArgument, ): """Parse a commandline.""" cmd_args = commandline_fluff parser.parse(cmd_args) assert not parser.config_file_data assert len(parser.cmd_priors) == 4 assert "/lr" in parser.cmd_priors assert "/prior" in parser.cmd_priors assert "/a.b" in parser.cmd_priors assert f"/{weird_argument.name}" in parser.cmd_priors assert parser.parser.template == [ "--seed", "{seed}", "--lr", "{lr}", "--non-prior", "{non-prior}", "--prior", "{prior}", "--a.b", "{a.b}", f"--{weird_argument.name}", f"{{{weird_argument.name}}}", "--some-path", "{some-path}", "--home-path", "{home-path[0]}", "{home-path[1]}", ]
def populate_priors(metadata): """Compute parser state and priors based on user_args and populate metadata.""" if "user_args" not in metadata: return update_user_args(metadata) parser = OrionCmdlineParser(orion.core.config.worker.user_script_config, allow_non_existing_files=True) if "parser" in metadata: # To keep configs like config user_script_config parser.config_prefix = metadata["parser"]["config_prefix"] parser.parse(metadata["user_args"]) log.debug("Updating parser for backward compatibility") metadata["parser"] = parser.get_state_dict() log.debug(pprint.pformat(metadata["parser"])) log.debug("Updating priors for backward compatibility") metadata["priors"] = dict(parser.priors) log.debug(pprint.pformat(metadata["priors"]))