def test_with_evc(algorithm):
"""Test a scenario where algos are warm-started with EVC."""
with OrionState(storage={
"type": "legacy",
"database": {
"type": "EphemeralDB"
}
}):
base_exp = create_experiment(
name="exp",
space=space_with_fidelity,
algorithms=algorithm_configs["random"],
)
base_exp.workon(rosenbrock, max_trials=10)
exp = create_experiment(
name="exp",
space=space_with_fidelity,
algorithms=algorithm,
branching={"branch_from": "exp"},
)
assert exp.version == 2
exp.workon(rosenbrock, max_trials=30)
assert exp.configuration["algorithms"] == algorithm
trials = exp.fetch_trials(with_evc_tree=False)
assert len(trials) >= 30
trials_with_evc = exp.fetch_trials(with_evc_tree=True)
assert len(trials_with_evc) >= 40
assert len(trials_with_evc) - len(trials) == 10
completed_trials = [
trial for trial in trials_with_evc if trial.status == "completed"
]
assert len(completed_trials) == 40
results = [trial.objective.value for trial in completed_trials]
best_trial = next(
iter(
sorted(completed_trials,
key=lambda trial: trial.objective.value)))
assert best_trial.objective.name == "objective"
assert abs(best_trial.objective.value - 23.4) < 1e-5
assert len(best_trial.params) == 2
fidelity = best_trial._params[0]
assert fidelity.name == "noise"
assert fidelity.type == "fidelity"
assert fidelity.value == 10
param = best_trial._params[1]
assert param.name == "x"
assert param.type == "real"
def setup_experiments(self):
"""Setup experiments to run of the study"""
max_trials = self.task.max_trials
task_num = self.assessment.task_num
space = self.task.get_search_space()
for task_index in range(task_num):
for algo_index, algorithm in enumerate(self.algorithms):
experiment_name = (
self.benchmark.name
+ "_"
+ self.assess_name
+ "_"
+ self.task_name
+ "_"
+ str(task_index)
+ "_"
+ str(algo_index)
)
experiment = create_experiment(
experiment_name,
space=space,
algorithms=algorithm,
max_trials=max_trials,
)
self.experiments_info.append((task_index, experiment))
def setup_experiments(self):
"""Setup experiments to run of the study"""
max_trials = self.task.max_trials
task_num = self.assessment.task_num
space = self.task.get_search_space()
for task_index in range(task_num):
for algo_index, algorithm in enumerate(self.algorithms):
# Run only 1 experiment for deterministic algorithm
if algorithm.is_deterministic and task_index > 0:
continue
experiment_name = (self.benchmark.name + "_" +
self.assess_name + "_" + self.task_name +
"_" + str(task_index) + "_" +
str(algo_index))
executor = (self.assessment.get_executor(task_index)
or self.benchmark.executor)
experiment = create_experiment(
experiment_name,
space=space,
algorithms=algorithm.experiment_algorithm,
max_trials=max_trials,
storage=self.benchmark.storage_config,
executor=executor,
)
self.experiments_info.append((task_index, experiment))
def test_hunt_python_api(self, fill_db):
"""Verify hunt command from python api parent"""
version = fill_db
if not has_python_api(version):
pytest.skip("Python API not supported by {}".format(version))
exp = create_experiment(
"hunt-python", branching={"branch-to": "hunt-python-branch"}
)
exp.workon(function, max_trials=10)
def test_result_reproducibility(monkeypatch):
"""Verifies the script results stays consistent (with respect to the documentation)."""
script = os.path.abspath("examples/scikitlearn-iris/main.py")
monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__)))
config = "orion_config.yaml"
orion.core.cli.main(
["hunt", "--config", config, "python", script, "orion~choices([0.1])"])
experiment = create_experiment(name="scikit-iris-tutorial")
assert experiment.stats is not None
assert experiment.stats.best_evaluation == 0.6666666666666667
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=64,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=1000,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=10,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=False,
help='For Saving the current Model')
args = parser.parse_args()
kwargs = vars(args)
experiment = create_experiment(name='mnist_example',
space={
'lr': 'loguniform(1e-5, 1.0)',
'momentum': 'uniform(0, 1)'
})
experiment.workon(run_trial, **kwargs)
def insert_trials(experiment_name, points, raise_exc=True):
"""Insert sets of parameters manually, defined in `points`, as new trials
for the experiment name, `experiment_name`.
.. warning::
This function is deprecated and will be removed in 0.3.0.
You should use ExperimentClient.insert() instead.
:param experiment_name: Name of the experiment which the new trials are
going to be associated with
:param points: list of tuples in agreement with experiment's parameter space
:param raise_exc: whether an inappropriate tuple of parameters will raise
an exception or it will be ignored
.. note:: If `raise_exc` is True, no set of parameters will be inserted. If
it is False, only the valid ones will be inserted; the rest will be ignored.
.. note:: This cannot be used to prepopulate a future experiment. So,
an experiment with `experiment_name` should already be configured in
the database.
"""
log.warning("insert_trials() is deprecated and will be removed in 0.3.0. "
"You should use ExperimentClient.insert() instead.")
experiment = create_experiment(experiment_name)
valid_points = []
for point in points:
try:
assert point in experiment.space
valid_points.append(point)
except AssertionError:
if raise_exc:
raise
if not valid_points:
return
new_trials = list(
map(
lambda data: format_trials.tuple_to_trial(data, experiment.space),
valid_points,
))
for new_trial in new_trials:
experiment.insert(new_trial.params)
def test_hunt_python_api(self, fill_db):
"""Verify hunt command from python api parent"""
version = fill_db
if version < '0.1.8':
pytest.skip("Python API not supported by {}".format(version))
def function(x):
"""Evaluate partial information of a quadratic."""
z = x - 34.56789
return [
dict(name='example_objective',
type='objective',
value=4 * z**2 + 23.4)
]
exp = create_experiment('hunt-python',
branching={'branch-to': 'hunt-python-branch'})
exp.workon(function, max_trials=10)
def test_optimizer_actually_optimize(monkeypatch):
"""Check if Bayesian Optimizer has better optimization than random search."""
monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__)))
best_random_search = 23.403275057472825
with OrionState(experiments=[], trials=[]):
orion.core.cli.main([
"hunt", "--name", "exp", "--max-trials", "20", "--config",
"./benchmark/robo.yaml", "./benchmark/rosenbrock.py",
"-x~uniform(-50, 50)"
])
exp = create_experiment(name="exp")
objective = exp.stats['best_evaluation']
assert best_random_search > objective
def get_experiment(storage, space_type, size):
"""Create an experiment or load from DB if already existing
Parameters
----------
storage: str
Can be `pickleddb` or `mongodb`. A default configuration is used for each.
space_type: str
Can be one of
- `discrete` Search space is discrete and limited to `max_trials`
- `real-seeded` Search space is continuous and algos is seeded, leading to many race
conditions while algos are sampling the same points in parallel, or
- `real` Search space is real and algo is not seeded, leading to very few race conditions.
size: int
This defines `max_trials`, and the size of the search space (`uniform(0, size)`).
"""
if storage == "pickleddb":
storage_config = {"type": "pickleddb", "host": DB_FILE}
elif storage == "mongodb":
storage_config = {
"type": "mongodb",
"name": "stress",
"host": "mongodb://*****:*****@localhost",
}
discrete = space_type == "discrete"
high = size # * 2
return create_experiment(
"stress-test",
space={"x": f"uniform(0, {high}, discrete={discrete})"},
max_trials=size,
max_idle_time=60 * 5,
algorithms={"random": {
"seed": None if space_type == "real" else 1
}},
storage={
"type": "legacy",
"database": storage_config
},
)
def hpo(n_workers=16):
experiment = create_experiment(
name="dask",
max_trials=1000,
max_broken=5,
space={
"C": "loguniform(1e-6, 1e6, precision=None)",
"gamma": "loguniform(1e-8, 1e8, precision=None)",
"tol": "loguniform(1e-4, 1e-1, precision=None)",
"class_weight": "choices([None, 'balanced'])",
},
algorithms={"random": {"seed": 1}},
)
with experiment.tmp_executor("dask", n_workers=n_workers):
experiment.workon(main, n_workers=n_workers // 2)
experiment.plot.regret().show()
experiment.plot.partial_dependencies(params=["C", "gamma", "tol"]).show()
def test_parallel_workers(algorithm):
"""Test parallel execution with joblib"""
MAX_TRIALS = 30
ASHA_UGLY_FIX = 10
with OrionState() as cfg: # Using PickledDB
name = f"{list(algorithm.keys())[0]}_exp"
exp = create_experiment(
name=name,
space=space_with_fidelity,
algorithms=algorithm,
)
exp.workon(rosenbrock, max_trials=MAX_TRIALS, n_workers=2)
assert exp.configuration["algorithms"] == algorithm
trials = exp.fetch_trials()
assert len(trials) >= MAX_TRIALS
completed_trials = [
trial for trial in trials if trial.status == "completed"
]
assert MAX_TRIALS <= len(completed_trials) <= MAX_TRIALS + 2
results = [trial.objective.value for trial in completed_trials]
assert all(trial.objective is not None for trial in completed_trials)
best_trial = min(completed_trials,
key=lambda trial: trial.objective.value)
assert best_trial.objective.name == "objective"
assert abs(best_trial.objective.value - 23.4) < 1e-5 + ASHA_UGLY_FIX
assert len(best_trial.params) == 2
fidelity = best_trial._params[0]
assert fidelity.name == "noise"
assert fidelity.type == "fidelity"
assert fidelity.value + ASHA_UGLY_FIX >= 1
param = best_trial._params[1]
assert param.name == "x"
assert param.type == "real"
def get_experiment(storage, space_type, size):
"""Create an experiment or load from DB if already existing
Parameters
----------
storage: str
Can be `pickleddb` or `mongodb`. A default configuration is used for each.
space_type: str
Can be one of
- `discrete` Search space is discrete and limited to `max_trials`
- `real-seeded` Search space is continuous and algos is seeded, leading to many race
conditions while algos are sampling the same points in parallel, or
- `real` Search space is real and algo is not seeded, leading to very few race conditions.
size: int
This defines `max_trials`, and the size of the search space (`uniform(0, size)`).
"""
if storage == 'pickleddb':
storage_config = {'type': 'pickleddb', 'host': DB_FILE}
elif storage == 'mongodb':
storage_config = {
'type': 'mongodb',
'name': 'stress',
'host': 'mongodb://*****:*****@localhost'
}
discrete = space_type == 'discrete'
high = size # * 2
return create_experiment(
'stress-test',
space={'x': f'uniform(0, {high}, discrete={discrete})'},
max_trials=size,
max_idle_time=60 * 5,
algorithms={'random': {
'seed': None if space_type == 'real' else 1
}},
storage={
'type': 'legacy',
'database': storage_config
})
def test_orion_runs_script(monkeypatch):
"""Verifies Oríon can execute the example script."""
script = os.path.abspath("examples/scikitlearn-iris/main.py")
monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__)))
config = "orion_config.yaml"
orion.core.cli.main(
["hunt", "--config", config, "python", script, "orion~choices([0.1])"])
experiment = create_experiment(name="scikit-iris-tutorial")
assert experiment is not None
assert experiment.version == 1
keys = experiment.space.keys()
assert len(keys) == 1
assert "/_pos_2" in keys
storage = get_storage()
trials = storage.fetch_trials(uid=experiment.id)
assert len(trials) == 1
trial = trials[0]
assert trial.status == "completed"
assert trial.params["/_pos_2"] == 0.1
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Simple example to fill db with python api"""
from orion.client import create_experiment
create_experiment(
"hunt-python",
space={"x": "uniform(-50,50)"},
algorithms={"random": {
"seed": 1
}},
max_trials=10,
)
create_experiment(
"hunt-python-branch-old",
space={"x": "uniform(-50,50)"},
algorithms={"random": {
"seed": 1
}},
branching={"branch_from": "hunt-python"},
)
def test_with_evc(algorithm):
"""Test a scenario where algos are warm-started with EVC."""
with OrionState(storage={
"type": "legacy",
"database": {
"type": "PickledDB"
}
}):
base_exp = create_experiment(
name="exp",
space=space_with_fidelity,
algorithms=algorithm_configs["random"],
max_trials=10,
)
base_exp.workon(rosenbrock, max_trials=10)
exp = create_experiment(
name="exp",
space=space_with_fidelity,
algorithms=algorithm,
max_trials=30,
branching={
"branch_from": "exp",
"enable": True
},
)
assert exp.version == 2
exp.workon(rosenbrock, max_trials=30)
assert exp.configuration["algorithms"] == algorithm
trials = exp.fetch_trials(with_evc_tree=False)
# Some algo may not be able to suggest exactly 30 trials (ex: hyperband)
assert len(trials) >= 20
trials_with_evc = exp.fetch_trials(with_evc_tree=True)
assert len(trials_with_evc) >= 30
assert len(trials_with_evc) - len(trials) == 10
completed_trials = [
trial for trial in trials_with_evc if trial.status == "completed"
]
assert len(completed_trials) == 30
results = [trial.objective.value for trial in completed_trials]
assert all(trial.objective is not None for trial in completed_trials)
best_trial = min(completed_trials,
key=lambda trial: trial.objective.value)
assert best_trial.objective.name == "objective"
assert abs(best_trial.objective.value - 23.4) < 1e-5
assert len(best_trial.params) == 2
fidelity = best_trial._params[0]
assert fidelity.name == "noise"
assert fidelity.type == "fidelity"
assert fidelity.value == 10
param = best_trial._params[1]
assert param.name == "x"
assert param.type == "real"
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=64,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=1000,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=10,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=False,
help='For Saving the current Model')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=args.batch_size,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=args.test_batch_size,
shuffle=True,
**kwargs)
# TODO: Use the actual framework API...
experiment = create_experiment(name='mnist_example',
space={
'lr': 'loguniform(1e-5, 1.0)',
'momentum': 'uniform(0, 1)'
})
trial = experiment.suggest()
params = trial.params
model = Net().to(device)
optimizer = optim.SGD(model.parameters(),
lr=params['lr'],
momentum=params['momentum'])
for epoch in range(1, args.epochs + 1):
train(args, model, device, train_loader, optimizer, epoch)
test_error_rate = test(args, model, device, test_loader)
experiment.observe(trial,
results=[
dict(name='test_error_rate',
type='objective',
value=test_error_rate)
])
if (args.save_model):
torch.save(model.state_dict(), "mnist_cnn.pt")
