def test_failure(): #XXX also test the Bandit.exceptions mechanism that actually catches them class BanditE(Exception): pass class DummyBandit(Bandit): param_gen = {"loss": 10} def __init__(self): super(DummyBandit, self).__init__(self.param_gen) def evaluate(self, config, ctrl): raise BanditE() trials = Trials() bandit_algo = Random(DummyBandit()) exp = Experiment(trials, bandit_algo, async=False) exp.run(1) trials.refresh() assert len(trials) == 0 assert len(trials._dynamic_trials) == 1 assert trials._dynamic_trials[0]['state'] == JOB_STATE_ERROR assert trials._dynamic_trials[0]['misc']['error'] != None exp.catch_bandit_exceptions = False nose.tools.assert_raises(BanditE, exp.run, 1) trials.refresh() assert len(trials) == 0 assert len(trials._dynamic_trials) == 2 assert trials._dynamic_trials[1]['state'] == JOB_STATE_ERROR assert trials._dynamic_trials[1]['misc']['error'] != None
class TestCoinFlipInjectorExperiment(unittest.TestCase): def setUp(self): self.bandit = CoinFlipInjector() self.algo = Random(self.bandit) self.trials = Trials() self.experiment = Experiment(self.trials, self.algo, async=False) def test_run_1(self): self.experiment.run(1) assert len(self.trials._trials) == 2
def foo(self): self.bandit = bandit = Bandit(self.expr) self.algo = algo = Random(bandit) if hasattr(self, 'n_randints'): n_randints = len(filter( lambda x: x.name == 'randint', algo.vh.params.values())) assert n_randints == self.n_randints self.trials = Trials() self.experiment = Experiment(self.trials, algo, async=False) self.experiment.run(5) self.output = output = [] for trial in self.trials._trials: print '' tmp = [] for nid in trial['misc']['idxs']: thing = ( nid, trial['misc']['idxs'][nid], trial['misc']['vals'][nid]) print thing tmp.append(thing) tmp.sort() output.append(tmp) print repr(output) print repr(self.wanted) # -- think of a more robust way to test these things # or, if the sampling style is to be nailed down, # put it in and be sure of it. raise nose.SkipTest() assert output == self.wanted
def foo(self): self.bandit = bandit = ZeroBandit(self.expr) self.algo = algo = Random(bandit) if hasattr(self, 'n_randints'): n_randints = len([nn for nn in algo.vh.name_by_id().values() if nn == 'randint']) assert n_randints == self.n_randints self.trials = trials = Trials() self.experiment = Experiment(trials, algo, async=False) self.experiment.run(5) trials = self.trials._trials self.output = output = [] for trial in trials: print '' tmp = [] for nid in trial['misc']['idxs']: thing = self.algo.doc_coords[nid], trial['misc']['idxs'][nid], trial['misc']['vals'][nid] print thing tmp.append(thing) tmp.sort() output.append(tmp) print repr(output) print repr(self.wanted) # -- think of a more robust way to test these things # or, if the sampling style is to be nailed down, # put it in and be sure of it. raise nose.SkipTest() assert output == self.wanted
class TestCoinFlipStopExperiment(unittest.TestCase): def setUp(self): self.bandit = coin_flip() self.algo = RandomStop(5, self.bandit) self.trials = Trials() self.experiment = Experiment(self.trials, self.algo, async=False) def test_run_1(self): self.experiment.run(1) assert len(self.trials._trials) == 1 def test_run_3_2_1_1(self): self.experiment.run(3) assert len(self.trials._trials) == 3 self.experiment.run(2) assert len(self.trials._trials) == 5 self.experiment.run(1) assert len(self.trials._trials) == 5 self.experiment.run(1) assert len(self.trials._trials) == 5
class TestCoinFlipExperiment(unittest.TestCase): def setUp(self): self.bandit = CoinFlip() self.algo = Random(self.bandit) self.trials = Trials() self.experiment = Experiment(self.trials, self.algo, async=False) def test_run_1(self): self.experiment.run(1) assert len(self.trials._trials) == 1 def test_run_1_1_1(self): self.experiment.run(1) self.experiment.run(1) self.experiment.run(1) assert len(self.trials._trials) == 3 print self.trials.miscs print self.trials.idxs print self.trials.vals assert self.trials.idxs['node_4'] == [0, 1, 2] assert self.trials.vals['node_4'] == [1, 1, 0]
class TestCoinFlipExperiment(unittest.TestCase): def setUp(self): self.bandit = coin_flip() self.algo = Random(self.bandit) self.trials = Trials() self.experiment = Experiment(self.trials, self.algo, async=False) def test_run_1(self): self.experiment.run(1) assert len(self.trials._trials) == 1 def test_run_1_1_1(self): self.experiment.run(1) self.experiment.run(1) self.experiment.run(1) assert len(self.trials._trials) == 3 print self.trials.miscs print self.trials.idxs print self.trials.vals assert self.trials.idxs['flip'] == [0, 1, 2] # -- assert that the random seed matches that of Jan 8/2013 assert self.trials.vals['flip'] == [0, 1, 0]
class TestConfigs(unittest.TestCase): def foo(self): self.bandit = bandit = Bandit(self.expr) self.algo = algo = Random(bandit) if hasattr(self, 'n_randints'): n_randints = len(filter( lambda x: x.name == 'randint', algo.vh.params.values())) assert n_randints == self.n_randints self.trials = Trials() self.experiment = Experiment(self.trials, algo, async=False) self.experiment.run(5) self.output = output = [] for trial in self.trials._trials: print '' tmp = [] for nid in trial['misc']['idxs']: thing = ( nid, trial['misc']['idxs'][nid], trial['misc']['vals'][nid]) print thing tmp.append(thing) tmp.sort() output.append(tmp) print repr(output) print repr(self.wanted) # -- think of a more robust way to test these things # or, if the sampling style is to be nailed down, # put it in and be sure of it. raise nose.SkipTest() assert output == self.wanted def test0(self): self.expr = {'loss': hp_uniform('p0', 0, 1)} self.wanted = [ [('p0', [0], [0.69646918559786164])], [('p0', [1], [0.28613933495037946])], [('p0', [2], [0.22685145356420311])], [('p0', [3], [0.55131476908289123])], [('p0', [4], [0.71946896978556307])]] self.foo() def test1(self): self.expr = as_apply(dict(p0=normal(0, 1))) self.wanted = [ [('p0', [0], [-1.0856306033005612])], [('p0', [1], [0.99734544658358582])], [('p0', [2], [0.28297849805199204])], [('p0', [3], [-1.506294713918092])], [('p0', [4], [-0.57860025196853637])]] self.foo() def test2(self): self.expr = as_apply(dict(p0=one_of(0, 1))) self.wanted = [ [('p0.randint', [0], [0])], [('p0.randint', [1], [1])], [('p0.randint', [2], [0])], [('p0.randint', [3], [0])], [('p0.randint', [4], [0])]] self.foo() def test3(self): self.expr = as_apply(dict(p0=uniform(0, 1), p1=normal(0, 1))) self.wanted = [ [('p0', [0], [0.69646918559786164]), ('p1', [0], [-0.95209720686132215])], [('p0', [1], [0.55131476908289123]), ('p1', [1], [-0.74544105948265826])], [('p0', [2], [0.71946896978556307]), ('p1', [2], [0.32210606833962163])], [('p0', [3], [0.68482973858486329]), ('p1', [3], [-0.0515177209393851])], [('p0', [4], [0.48093190148436094]), ('p1', [4], [-1.6193000650367457])]] self.foo() def test4(self): self.expr = as_apply(dict(p0=uniform(0, 1) + normal(0, 1))) self.wanted = [ [('p0.arg:0', [0], [0.69646918559786164]), ('p0.arg:1', [0], [-0.95209720686132215])], [('p0.arg:0', [1], [0.55131476908289123]), ('p0.arg:1', [1], [-0.74544105948265826])], [('p0.arg:0', [2], [0.71946896978556307]), ('p0.arg:1', [2], [0.32210606833962163])], [('p0.arg:0', [3], [0.68482973858486329]), ('p0.arg:1', [3], [-0.0515177209393851])], [('p0.arg:0', [4], [0.48093190148436094]), ('p0.arg:1', [4], [-1.6193000650367457])]] self.foo() def test5(self): p0 = uniform(0, 1) self.expr = as_apply(dict(p0=p0, p1=p0)) self.wanted = [[('p0', [0], [0.69646918559786164])], [('p0', [1], [0.28613933495037946])], [('p0', [2], [0.22685145356420311])], [('p0', [3], [0.55131476908289123])], [('p0', [4], [0.71946896978556307])]] self.foo() def test6(self): p0 = uniform(0, 1) self.expr = as_apply(dict(p0=p0, p1=normal(p0, 1))) self.wanted = [ [('p0', [0], [0.69646918559786164]), ('p1', [0], [-0.25562802126346051])], [('p0', [1], [0.55131476908289123]), ('p1', [1], [-0.19412629039976703])], [('p0', [2], [0.71946896978556307]), ('p1', [2], [1.0415750381251847])], [('p0', [3], [0.68482973858486329]), ('p1', [3], [0.63331201764547818])], [('p0', [4], [0.48093190148436094]), ('p1', [4], [-1.1383681635523848])]] self.foo() def test7(self): p0 = hp_uniform('p0', 0, 1) p1 = hp_normal('p1', 0, 1) p2 = hp_choice('p2', [1, p0]) p3 = hp_choice('p3', [2, p1, p2, hp_uniform('a0', 2, 3)]) self.expr = {'loss': p0 + p1 + p2 + p3} self.n_randints = 2 self.wanted = [ [ ('p0', [0], [0.71295532052322719]), ('p1', [0], [0.28297849805199204]), ('p2.randint', [0], [0]), ('p3.arg:2', [0], [2.719468969785563]), ('p3.randint', [0], [2])], [ ('p0', [1], [0.78002776191207912]), ('p1', [1], [-1.506294713918092]), ('p2.randint', [1], [1]), ('p3.arg:2', [], []), ('p3.randint', [1], [1])], [ ('p0', [2], [0.57969429702261011]), ('p1', [2], [1.6796003743035337]), ('p2.randint', [2], [0]), ('p3.arg:2', [], []), ('p3.randint', [2], [1])], [ ('p0', [3], [0.43857224467962441]), ('p1', [3], [-1.3058031267484451]), ('p2.randint', [3], [1]), ('p3.arg:2', [], []), ('p3.randint', [3], [1])], [ ('p0', [4], [0.39804425533043142]), ('p1', [4], [-0.91948540682140967]), ('p2.randint', [4], [0]), ('p3.arg:2', [], []), ('p3.randint', [4], [0])]] self.foo()
def setUp(self): self.bandit = coin_flip() self.algo = RandomStop(5, self.bandit) self.trials = Trials() self.experiment = Experiment(self.trials, self.algo, async=False)
def setUp(self): self.bandit = CoinFlipInjector() self.algo = Random(self.bandit) self.trials = Trials() self.experiment = Experiment(self.trials, self.algo, async=False)