def test_metric_invalid_trial_id(self): ex = daskperiment.Experiment(id="test_metric_invalid", backend=self.backend) a = ex.parameter('a') assert ex.trial_id == 0 @ex.result def inc(a): ex.save_metric('dummy_metric', epoch=1, value=2) ex.save_metric('dummy_metric', epoch=2, value=4) return a + 1 res = inc(a) ex.set_parameters(a=3) assert res.compute() == 4 assert ex.trial_id == 1 match = 'Unable to find trial id:' with pytest.raises(TrialIDNotFoundError, match=match): ex.load_metric('dummy_metric', trial_id=0) with pytest.raises(TrialIDNotFoundError, match=match): ex.load_metric('dummy_metric', trial_id=2) res = ex.load_metric('dummy_metric', trial_id=1) exp_idx = pd.Index([1, 2], name='Epoch') exp_columns = pd.Index([1], name='Trial ID') exp = pd.DataFrame({1: [2, 4]}, index=exp_idx, columns=exp_columns) tm.assert_frame_equal(res, exp) with pytest.raises(ValueError, match=('Unable to find saved metric ' 'with specified key:')): ex.load_metric('invalid_metric', trial_id=1) @ex.result def inc(a): return a + 1 res = inc(a) ex.set_parameters(a=4) assert res.compute() == 5 assert ex.trial_id == 2 with pytest.raises(TrialIDNotFoundError, match=match): ex.load_metric('dummy_metric', trial_id=2)
def setup_class(cls): # TODO: setup_class ex = daskperiment.Experiment('test_board') a = ex.parameter('a') @ex.result def inc(a): for i in range(3): ex.save_metric('my_metric', epoch=i, value=a + i) return a + 1 res = inc(a) for i in range(3): ex.set_parameters(a=i) res.compute() board.ex = ex cls.ex = ex
def test_random_experiment_no_params(self): file = 'scripts/random_experiment.py' p = subprocess.Popen([self.exe, file], stdout=subprocess.PIPE) stdout, stderr = p.communicate() assert p.returncode == 0 # provide seed p = subprocess.Popen([self.exe, file, '--seed', '1'], stdout=subprocess.PIPE) stdout, stderr = p.communicate() assert p.returncode == 0 # cleanup e = daskperiment.Experiment('random_experiment_pj') h = e.get_history() assert h.loc[1, 'Result'] != 0.5513862488149752 assert h.loc[2, 'Result'] == 0.5513862488149752 e._delete_cache()
def test_internal(self): ex = daskperiment.Experiment(id="test_internal") a = ex.parameter("a") assert a._value == Undefined() assert a.is_undefined with pytest.raises(daskperiment.core.errors.ParameterUndefinedError): assert a.compute() with pytest.raises(daskperiment.core.errors.ParameterUndefinedError): assert a.resolve() assert a.resolve(allow_undefined=True) == Undefined() assert repr(a) == "Parameter(a: Undefined)" assert a.summarize() == "a=Undefined" a.set('aaa') assert a._value == 'aaa' assert not a.is_undefined assert a.compute() == 'aaa' assert a.resolve() == 'aaa' assert repr(a) == "Parameter(a: aaa<class 'str'>)" assert a.summarize() == "a=aaa<class 'str'>" ex._delete_cache()
def test_build_layout(self): ex = daskperiment.Experiment('board') # empty experiment b = DaskperimentBoard(ex) app = b.build_layout() assert isinstance(app, Column), type(app) a = ex.parameter('a') @ex.result def inc(a): return a + 1 res = inc(a) ex.set_parameters(a=1) assert res.compute() == 2 b = DaskperimentBoard(ex) app = b.build_layout() assert isinstance(app, Column), type(app) @ex.result def inc(a): ex.save_metric('dummy', epoch=1, value=1) ex.save_metric('dummy2', epoch=1, value=1) return a + 1 res = inc(a) ex.set_parameters(a=2) assert res.compute() == 3 b = DaskperimentBoard(ex) app = b.build_layout() assert isinstance(app, Column), type(app) ex._delete_cache()
import daskperiment ex = daskperiment.Experiment(id='simple_experiment_pj') a = ex.parameter('a') b = ex.parameter('b') @ex def prepare_data(a, b): return a + b @ex.result def calculate_score(s): return s + 1 d = prepare_data(a, b) calculate_score(d)
server = Server(apps, port=port, extra_patterns=patterns) if experiment._environment.maybe_jupyter(): # in jupyter, .run_until_shutdown() may raise RuntimeError # because of IOLoop server.start() else: server.run_until_shutdown() DaskperimentBoard._STARTED = True return server if __name__ == '__main__': ex = daskperiment.Experiment('bokeh_test') a = ex.parameter('a') @ex.result def inc(a): for i in range(100): ex.save_metric('dummy_score', epoch=i, value=100 + np.random.random()) ex.save_metric('dummy_score2', epoch=i, value=10) return a + 1 ex.set_parameters(a=2) inc(a) ex.start_dashboard()
def test_is(self): ex1 = daskperiment.Experiment(id="test_parameter", backend=self.backend) ex2 = daskperiment.Experiment(id="test_parameter", backend=self.backend) assert ex1 is ex2
def test_metric_experiment(self): ex = daskperiment.Experiment('metric_experiment', backend=self.backend) a = ex.parameter('a') @ex.result def dummy(a): ex.save_metric('exp_metric1', epoch=1, value=a + 1) ex.save_metric('exp_metric1', epoch=3, value=a + 2) ex.save_metric('exp_metric2', epoch=1, value=a + 11) ex.save_metric('exp_metric2', epoch=3, value=a + 12) return a + 1 res = dummy(a) ex.set_parameters(a=1) assert res.compute() == 2 ex.set_parameters(a=2) assert res.compute() == 3 assert ex._metrics.keys() == ['exp_metric1', 'exp_metric2'] res = ex.load_metric('exp_metric1', trial_id=[1, 2]) exp_idx = pd.Index([1, 3], name='Epoch') exp_columns = pd.Index([1, 2], name='Trial ID') exp = pd.DataFrame({ 1: [2, 3], 2: [3, 4] }, index=exp_idx, columns=exp_columns) tm.assert_frame_equal(res, exp) res = ex.load_metric('exp_metric2', trial_id=[1, 2]) exp_idx = pd.Index([1, 3], name='Epoch') exp_columns = pd.Index([1, 2], name='Trial ID') exp = pd.DataFrame({ 1: [12, 13], 2: [13, 14] }, index=exp_idx, columns=exp_columns) tm.assert_frame_equal(res, exp) # check with another instance del daskperiment.Experiment._instance_cache['metric_experiment'] ex = daskperiment.Experiment('metric_experiment', backend=self.backend) assert ex._metrics.keys() == ['exp_metric1', 'exp_metric2'] res = ex.load_metric('exp_metric1', trial_id=[1, 2]) exp_idx = pd.Index([1, 3], name='Epoch') exp_columns = pd.Index([1, 2], name='Trial ID') exp = pd.DataFrame({ 1: [2, 3], 2: [3, 4] }, index=exp_idx, columns=exp_columns) tm.assert_frame_equal(res, exp) res = ex.load_metric('exp_metric2', trial_id=[1, 2]) exp_idx = pd.Index([1, 3], name='Epoch') exp_columns = pd.Index([1, 2], name='Trial ID') exp = pd.DataFrame({ 1: [12, 13], 2: [13, 14] }, index=exp_idx, columns=exp_columns) tm.assert_frame_equal(res, exp) ex._delete_cache()
def test_invalid_id(self, key): msg = 'Experiment ID must be ' with pytest.raises(ValueError, match=msg): daskperiment.Experiment(id=key, backend=self.backend)
def board(port, experiment): ex = daskperiment.Experiment(experiment) ex.start_dashboard(port=port)
import random import numpy as np import daskperiment ex = daskperiment.Experiment(id='random_experiment_pj') @ex.result def random_result(): res = np.random.random() + random.random() print(res) return res random_result() print(ex.get_history())