def test_build_from_args_debug_mode(script_path): """Try building experiment in debug mode""" update_singletons() experiment_builder.build_from_args({ 'name': 'whatever', 'user_args': [script_path, '--mini-batch~uniform(32, 256)'] }) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) update_singletons() experiment_builder.build_from_args({ 'name': 'whatever', 'user_args': [script_path, '--mini-batch~uniform(32, 256)'], 'debug': True }) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, EphemeralDB)
def test_build_from_args_debug_mode(script_path): """Try building experiment in debug mode""" update_singletons() experiment_builder.build_from_args( { "name": "whatever", "user_args": [script_path, "--mini-batch~uniform(32, 256)"], } ) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) update_singletons() experiment_builder.build_from_args( { "name": "whatever", "user_args": [script_path, "--mini-batch~uniform(32, 256)"], "debug": True, } ) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, EphemeralDB)
def main(args): """Build experiment and execute hunt command""" args["root"] = None args["leafs"] = [] # TODO: simplify when parameter parsing is refactored experiment = experiment_builder.build_from_args(args) if args["init_only"]: return config = experiment_builder.get_cmd_config(args) worker_config = orion.core.config.worker.to_dict() if config.get("worker"): worker_config.update(config.get("worker")) # If EVC is not enabled, we force Consumer to ignore code changes. if not config["branching"].get("enable", orion.core.config.evc.enable): ignore_code_changes = True else: ignore_code_changes = config["branching"].get("ignore_code_changes") with sigterm_as_interrupt(): workon(experiment, ignore_code_changes=ignore_code_changes, **worker_config)
def test_build_from_args_no_hit(config_file, random_dt, script_path, new_config): """Try building experiment when not in db""" cmdargs = { "name": "supernaekei", "config": config_file, "user_args": [script_path, "x~uniform(0,10)"], } with OrionState(experiments=[], trials=[]): with pytest.raises(NoConfigurationError) as exc_info: experiment_builder.get_from_args(cmdargs) assert "No experiment with given name 'supernaekei' and version '*'" in str( exc_info.value ) exp = experiment_builder.build_from_args(cmdargs) assert exp.name == cmdargs["name"] assert exp.configuration["refers"] == { "adapter": [], "parent_id": None, "root_id": exp._id, } assert exp.metadata["datetime"] == random_dt assert exp.metadata["user"] == "dendi" assert exp.metadata["user_script"] == cmdargs["user_args"][0] assert exp.metadata["user_args"] == cmdargs["user_args"] assert exp.max_trials == 100 assert exp.max_broken == 5 assert exp.algorithms.configuration == {"random": {"seed": None}}
def test_build_from_args_no_hit(config_file, random_dt, script_path, new_config): """Try building experiment when not in db""" cmdargs = { 'name': 'supernaekei', 'config': config_file, 'user_args': [script_path, 'x~uniform(0,10)'] } with OrionState(experiments=[], trials=[]): with pytest.raises(ValueError) as exc_info: experiment_builder.build_view_from_args(cmdargs) assert "No experiment with given name 'supernaekei' and version '*'" in str( exc_info.value) exp = experiment_builder.build_from_args(cmdargs) assert exp.name == cmdargs['name'] assert exp.configuration['refers'] == { 'adapter': [], 'parent_id': None, 'root_id': exp._id } assert exp.metadata['datetime'] == random_dt assert exp.metadata['user'] == 'dendi' assert exp.metadata['user_script'] == cmdargs['user_args'][0] assert exp.metadata['user_args'] == cmdargs['user_args'] assert exp.pool_size == 1 assert exp.max_trials == 100 assert exp.algorithms.configuration == {'random': {'seed': None}}
def test_build_from_args_force_user(new_config): """Try building experiment view when in db""" cmdargs = {"name": new_config["name"]} cmdargs["user"] = "******" with OrionState(experiments=[new_config], trials=[]): # Test that experiment already exists exp_view = experiment_builder.build_from_args(cmdargs) assert exp_view.metadata["user"] == "tsirif"
def test_build_from_args_force_user(new_config): """Try building experiment view when in db""" cmdargs = {'name': new_config['name']} cmdargs['user'] = '******' with OrionState(experiments=[new_config], trials=[]): # Test that experiment already exists exp_view = experiment_builder.build_from_args(cmdargs) assert exp_view.metadata['user'] == 'tsirif'
def test_workon(): """Test scenario having a configured experiment already setup.""" name = "voici_voila" config = {"name": name} config["algorithms"] = {"gradient_descent": {"learning_rate": 0.1}} config["pool_size"] = 1 config["max_trials"] = 100 config["exp_max_broken"] = 5 config["user_args"] = [ os.path.abspath(os.path.join(os.path.dirname(__file__), "black_box.py")), "-x~uniform(-50, 50, precision=None)", ] with OrionState(): experiment = experiment_builder.build_from_args(config) workon(experiment, 100, 100, 100, 100, 100) storage = get_storage() exp = list(storage.fetch_experiments({"name": name})) assert len(exp) == 1 exp = exp[0] assert "_id" in exp assert exp["name"] == name assert exp["pool_size"] == 1 assert exp["max_trials"] == 100 assert exp["max_broken"] == 5 assert exp["algorithms"] == { "gradient_descent": { "learning_rate": 0.1, "dx_tolerance": 1e-7 } } assert "user" in exp["metadata"] assert "datetime" in exp["metadata"] assert "user_script" in exp["metadata"] assert exp["metadata"]["user_args"] == config["user_args"] trials = list(storage.fetch_trials(experiment)) assert len(trials) <= 15 trials = list(sorted(trials, key=lambda trial: trial.submit_time)) assert trials[-1].status == "completed" for result in trials[-1].results: assert result.type != "constraint" if result.type == "objective": assert abs(result.value - 23.4) < 1e-6 assert result.name == "example_objective" elif result.type == "gradient": res = numpy.asarray(result.value) assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7 assert result.name == "example_gradient" params = trials[-1].params assert len(params) == 1 px = params["/x"] assert isinstance(px, float) assert (px - 34.56789) < 1e-5
def test_workon(): """Test scenario having a configured experiment already setup.""" name = "voici_voila" config = {"name": name} config["algorithms"] = {"random": {"seed": 1}} config["max_trials"] = 50 config["exp_max_broken"] = 5 config["user_args"] = [ os.path.abspath(os.path.join(os.path.dirname(__file__), "black_box.py")), "-x~uniform(-50, 50, precision=None)", ] with OrionState(): experiment = experiment_builder.build_from_args(config) workon( experiment, n_workers=2, max_trials=10, max_broken=5, max_idle_time=20, heartbeat=20, user_script_config="config", interrupt_signal_code=120, ignore_code_changes=True, executor="joblib", executor_configuration={"backend": "threading"}, ) storage = get_storage() exp = list(storage.fetch_experiments({"name": name})) assert len(exp) == 1 exp = exp[0] assert "_id" in exp assert exp["name"] == name assert exp["max_trials"] == 50 assert exp["max_broken"] == 5 assert exp["algorithms"] == {"random": {"seed": 1}} assert "user" in exp["metadata"] assert "datetime" in exp["metadata"] assert "user_script" in exp["metadata"] assert exp["metadata"]["user_args"] == config["user_args"] trials = experiment.fetch_trials_by_status("completed") assert len(trials) <= 22 trials = list(sorted(trials, key=lambda trial: trial.submit_time)) assert trials[-1].status == "completed" params = trials[-1].params assert len(params) == 1 px = params["/x"] assert isinstance(px, float) assert (px - 34.56789) < 20
def test_workon(): """Test scenario having a configured experiment already setup.""" name = 'voici_voila' config = {'name': name} config['algorithms'] = { 'gradient_descent': { 'learning_rate': 0.1 } } config['pool_size'] = 1 config['max_trials'] = 100 config['user_args'] = [ os.path.abspath(os.path.join(os.path.dirname(__file__), "black_box.py")), "-x~uniform(-50, 50, precision=None)"] with OrionState(): experiment = experiment_builder.build_from_args(config) workon(experiment, 100, 100, 100, 100, 100) storage = get_storage() exp = list(storage.fetch_experiments({'name': name})) assert len(exp) == 1 exp = exp[0] assert '_id' in exp assert exp['name'] == name assert exp['pool_size'] == 1 assert exp['max_trials'] == 100 assert exp['algorithms'] == {'gradient_descent': {'learning_rate': 0.1, 'dx_tolerance': 1e-7}} assert 'user' in exp['metadata'] assert 'datetime' in exp['metadata'] assert 'user_script' in exp['metadata'] assert exp['metadata']['user_args'] == config['user_args'] trials = list(storage.fetch_trials(experiment)) assert len(trials) <= 15 trials = list(sorted(trials, key=lambda trial: trial.submit_time)) assert trials[-1].status == 'completed' for result in trials[-1].results: assert result.type != 'constraint' if result.type == 'objective': assert abs(result.value - 23.4) < 1e-6 assert result.name == 'example_objective' elif result.type == 'gradient': res = numpy.asarray(result.value) assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7 assert result.name == 'example_gradient' params = trials[-1]._params assert len(params) == 1 assert params[0].name == '/x' assert params[0].type == 'real' assert (params[0].value - 34.56789) < 1e-5
def main(args): """Build experiment and execute hunt command""" args['root'] = None args['leafs'] = [] # TODO: simplify when parameter parsing is refactored experiment = experiment_builder.build_from_args(args) config = experiment_builder.get_cmd_config(args) worker_config = orion.core.config.worker.to_dict() if config.get('worker'): worker_config.update(config.get('worker')) workon(experiment, **worker_config)
def test_build_from_args_without_cmd(old_config_file, script_path, new_config): """Try building experiment without commandline when in db (no branch)""" name = 'supernaekei' cmdargs = {'name': name, 'config': old_config_file} with OrionState(experiments=[new_config], trials=[]): # Test that experiment already exists (this should fail otherwise) experiment_builder.build_view_from_args(cmdargs) exp = experiment_builder.build_from_args(cmdargs) assert exp._id == new_config['_id'] assert exp.name == new_config['name'] assert exp.configuration['refers'] == new_config['refers'] assert exp.metadata == new_config['metadata'] assert exp.max_trials == new_config['max_trials'] assert exp.algorithms.configuration == new_config['algorithms']
def main(args): """Build experiment and execute hunt command""" args["root"] = None args["leafs"] = [] # TODO: simplify when parameter parsing is refactored experiment = experiment_builder.build_from_args(args) if args["init_only"]: return config = experiment_builder.get_cmd_config(args) worker_config = orion.core.config.worker.to_dict() if config.get("worker"): worker_config.update(config.get("worker")) workon(experiment, ignore_code_changes=config["branching"].get("ignore_code_changes"), **worker_config)
def test_build_from_args_without_cmd(old_config_file, script_path, new_config): """Try building experiment without commandline when in db (no branch)""" name = "supernaekei" cmdargs = {"name": name, "config": old_config_file} with OrionState(experiments=[new_config], trials=[]): # Test that experiment already exists (this should fail otherwise) experiment_builder.get_from_args(cmdargs) exp = experiment_builder.build_from_args(cmdargs) assert exp._id == new_config["_id"] assert exp.name == new_config["name"] assert exp.configuration["refers"] == new_config["refers"] assert exp.metadata == new_config["metadata"] assert exp.max_trials == new_config["max_trials"] assert exp.max_broken == new_config["max_broken"] assert exp.algorithms.configuration == new_config["algorithms"]
def test_build_from_args_hit(old_config_file, script_path, new_config): """Try building experiment when in db (no branch)""" cmdargs = { 'name': 'supernaekei', 'config': old_config_file, 'user_args': [script_path, '--mini-batch~uniform(32, 256, discrete=True)'] } with OrionState(experiments=[new_config], trials=[]): # Test that experiment already exists experiment_builder.build_view_from_args(cmdargs) exp = experiment_builder.build_from_args(cmdargs) assert exp._id == new_config['_id'] assert exp.name == new_config['name'] assert exp.version == 1 assert exp.configuration['refers'] == new_config['refers'] assert exp.metadata == new_config['metadata'] assert exp.max_trials == new_config['max_trials'] assert exp.algorithms.configuration == new_config['algorithms']
def test_build_from_args_hit(old_config_file, script_path, new_config): """Try building experiment when in db (no branch)""" cmdargs = { "name": "supernaekei", "config": old_config_file, "user_args": [script_path, "--mini-batch~uniform(32, 256, discrete=True)"], } with OrionState(experiments=[new_config], trials=[]): # Test that experiment already exists experiment_builder.get_from_args(cmdargs) exp = experiment_builder.build_from_args(cmdargs) assert exp._id == new_config["_id"] assert exp.name == new_config["name"] assert exp.version == 1 assert exp.configuration["refers"] == new_config["refers"] assert exp.metadata == new_config["metadata"] assert exp.max_trials == new_config["max_trials"] assert exp.max_broken == new_config["max_broken"] assert exp.algorithms.configuration == new_config["algorithms"]
def main(args): """Build and initialize experiment""" # By building the experiment, we create a new experiment document in database log.warning("Command init_only is deprecated and will be removed in v0.3. " "Use orion hunt --init-only instead.") experiment_builder.build_from_args(args)
def main(args): """Build and initialize experiment""" # By building the experiment, we create a new experiment document in database experiment_builder.build_from_args(args)