def test_metric_invalid_trial_id(self):
ex = daskperiment.Experiment(id="test_metric_invalid",
backend=self.backend)
a = ex.parameter('a')
assert ex.trial_id == 0
@ex.result
def inc(a):
ex.save_metric('dummy_metric', epoch=1, value=2)
ex.save_metric('dummy_metric', epoch=2, value=4)
return a + 1
res = inc(a)
ex.set_parameters(a=3)
assert res.compute() == 4
assert ex.trial_id == 1
match = 'Unable to find trial id:'
with pytest.raises(TrialIDNotFoundError, match=match):
ex.load_metric('dummy_metric', trial_id=0)
with pytest.raises(TrialIDNotFoundError, match=match):
ex.load_metric('dummy_metric', trial_id=2)
res = ex.load_metric('dummy_metric', trial_id=1)
exp_idx = pd.Index([1, 2], name='Epoch')
exp_columns = pd.Index([1], name='Trial ID')
exp = pd.DataFrame({1: [2, 4]}, index=exp_idx, columns=exp_columns)
tm.assert_frame_equal(res, exp)
with pytest.raises(ValueError,
match=('Unable to find saved metric '
'with specified key:')):
ex.load_metric('invalid_metric', trial_id=1)
@ex.result
def inc(a):
return a + 1
res = inc(a)
ex.set_parameters(a=4)
assert res.compute() == 5
assert ex.trial_id == 2
with pytest.raises(TrialIDNotFoundError, match=match):
ex.load_metric('dummy_metric', trial_id=2)
def setup_class(cls):
# TODO: setup_class
ex = daskperiment.Experiment('test_board')
a = ex.parameter('a')
@ex.result
def inc(a):
for i in range(3):
ex.save_metric('my_metric', epoch=i, value=a + i)
return a + 1
res = inc(a)
for i in range(3):
ex.set_parameters(a=i)
res.compute()
board.ex = ex
cls.ex = ex
def test_random_experiment_no_params(self):
file = 'scripts/random_experiment.py'
p = subprocess.Popen([self.exe, file], stdout=subprocess.PIPE)
stdout, stderr = p.communicate()
assert p.returncode == 0
# provide seed
p = subprocess.Popen([self.exe, file, '--seed', '1'],
stdout=subprocess.PIPE)
stdout, stderr = p.communicate()
assert p.returncode == 0
# cleanup
e = daskperiment.Experiment('random_experiment_pj')
h = e.get_history()
assert h.loc[1, 'Result'] != 0.5513862488149752
assert h.loc[2, 'Result'] == 0.5513862488149752
e._delete_cache()
def test_internal(self):
ex = daskperiment.Experiment(id="test_internal")
a = ex.parameter("a")
assert a._value == Undefined()
assert a.is_undefined
with pytest.raises(daskperiment.core.errors.ParameterUndefinedError):
assert a.compute()
with pytest.raises(daskperiment.core.errors.ParameterUndefinedError):
assert a.resolve()
assert a.resolve(allow_undefined=True) == Undefined()
assert repr(a) == "Parameter(a: Undefined)"
assert a.summarize() == "a=Undefined"
a.set('aaa')
assert a._value == 'aaa'
assert not a.is_undefined
assert a.compute() == 'aaa'
assert a.resolve() == 'aaa'
assert repr(a) == "Parameter(a: aaa<class 'str'>)"
assert a.summarize() == "a=aaa<class 'str'>"
ex._delete_cache()
def test_build_layout(self):
ex = daskperiment.Experiment('board')
# empty experiment
b = DaskperimentBoard(ex)
app = b.build_layout()
assert isinstance(app, Column), type(app)
a = ex.parameter('a')
@ex.result
def inc(a):
return a + 1
res = inc(a)
ex.set_parameters(a=1)
assert res.compute() == 2
b = DaskperimentBoard(ex)
app = b.build_layout()
assert isinstance(app, Column), type(app)
@ex.result
def inc(a):
ex.save_metric('dummy', epoch=1, value=1)
ex.save_metric('dummy2', epoch=1, value=1)
return a + 1
res = inc(a)
ex.set_parameters(a=2)
assert res.compute() == 3
b = DaskperimentBoard(ex)
app = b.build_layout()
assert isinstance(app, Column), type(app)
ex._delete_cache()
import daskperiment
ex = daskperiment.Experiment(id='simple_experiment_pj')
a = ex.parameter('a')
b = ex.parameter('b')
@ex
def prepare_data(a, b):
return a + b
@ex.result
def calculate_score(s):
return s + 1
d = prepare_data(a, b)
calculate_score(d)
server = Server(apps, port=port, extra_patterns=patterns)
if experiment._environment.maybe_jupyter():
# in jupyter, .run_until_shutdown() may raise RuntimeError
# because of IOLoop
server.start()
else:
server.run_until_shutdown()
DaskperimentBoard._STARTED = True
return server
if __name__ == '__main__':
ex = daskperiment.Experiment('bokeh_test')
a = ex.parameter('a')
@ex.result
def inc(a):
for i in range(100):
ex.save_metric('dummy_score', epoch=i,
value=100 + np.random.random())
ex.save_metric('dummy_score2', epoch=i, value=10)
return a + 1
ex.set_parameters(a=2)
inc(a)
ex.start_dashboard()
def test_is(self):
ex1 = daskperiment.Experiment(id="test_parameter",
backend=self.backend)
ex2 = daskperiment.Experiment(id="test_parameter",
backend=self.backend)
assert ex1 is ex2
def test_metric_experiment(self):
ex = daskperiment.Experiment('metric_experiment', backend=self.backend)
a = ex.parameter('a')
@ex.result
def dummy(a):
ex.save_metric('exp_metric1', epoch=1, value=a + 1)
ex.save_metric('exp_metric1', epoch=3, value=a + 2)
ex.save_metric('exp_metric2', epoch=1, value=a + 11)
ex.save_metric('exp_metric2', epoch=3, value=a + 12)
return a + 1
res = dummy(a)
ex.set_parameters(a=1)
assert res.compute() == 2
ex.set_parameters(a=2)
assert res.compute() == 3
assert ex._metrics.keys() == ['exp_metric1', 'exp_metric2']
res = ex.load_metric('exp_metric1', trial_id=[1, 2])
exp_idx = pd.Index([1, 3], name='Epoch')
exp_columns = pd.Index([1, 2], name='Trial ID')
exp = pd.DataFrame({
1: [2, 3],
2: [3, 4]
},
index=exp_idx,
columns=exp_columns)
tm.assert_frame_equal(res, exp)
res = ex.load_metric('exp_metric2', trial_id=[1, 2])
exp_idx = pd.Index([1, 3], name='Epoch')
exp_columns = pd.Index([1, 2], name='Trial ID')
exp = pd.DataFrame({
1: [12, 13],
2: [13, 14]
},
index=exp_idx,
columns=exp_columns)
tm.assert_frame_equal(res, exp)
# check with another instance
del daskperiment.Experiment._instance_cache['metric_experiment']
ex = daskperiment.Experiment('metric_experiment', backend=self.backend)
assert ex._metrics.keys() == ['exp_metric1', 'exp_metric2']
res = ex.load_metric('exp_metric1', trial_id=[1, 2])
exp_idx = pd.Index([1, 3], name='Epoch')
exp_columns = pd.Index([1, 2], name='Trial ID')
exp = pd.DataFrame({
1: [2, 3],
2: [3, 4]
},
index=exp_idx,
columns=exp_columns)
tm.assert_frame_equal(res, exp)
res = ex.load_metric('exp_metric2', trial_id=[1, 2])
exp_idx = pd.Index([1, 3], name='Epoch')
exp_columns = pd.Index([1, 2], name='Trial ID')
exp = pd.DataFrame({
1: [12, 13],
2: [13, 14]
},
index=exp_idx,
columns=exp_columns)
tm.assert_frame_equal(res, exp)
ex._delete_cache()
def test_invalid_id(self, key):
msg = 'Experiment ID must be '
with pytest.raises(ValueError, match=msg):
daskperiment.Experiment(id=key, backend=self.backend)
def board(port, experiment):
ex = daskperiment.Experiment(experiment)
ex.start_dashboard(port=port)
import random
import numpy as np
import daskperiment
ex = daskperiment.Experiment(id='random_experiment_pj')
@ex.result
def random_result():
res = np.random.random() + random.random()
print(res)
return res
random_result()
print(ex.get_history())
