class TestRandom(unittest.TestCase):
def setUp(self):
self.bandit = coin_flip()
self.algo = Random(self.bandit)
def test_suggest_1(self):
print 'EXPR', self.bandit.expr
docs = self.algo.suggest([0], Trials())
assert len(docs) == 1
print 'DOCS', docs
# -- assert validity of docs
trials = trials_from_docs(docs)
print 'TRIALS', trials
assert docs[0]['misc']['idxs']['flip'] == [0]
idxs, vals = miscs_to_idxs_vals(trials.miscs)
assert idxs['flip'] == [0]
def test_suggest_N(self, N=10):
assert N <= 10
docs = self.algo.suggest(range(N), Trials())
print 'docs', docs
assert len(docs) == N
# -- assert validity of docs
trials = trials_from_docs(docs)
idxs, vals = miscs_to_idxs_vals(trials.miscs)
print 'idxs', idxs
print 'vals', vals
assert len(idxs) == 1
assert len(vals) == 1
assert idxs['flip'] == range(N)
# -- only works when N == 5
assert np.all(vals['flip'] == [0, 1, 0, 0, 0, 0, 0, 1, 1, 0][:N])
def test_arbitrary_range(self, N=10):
assert N <= 10
new_ids = [-2, 0, 7, 'a', '007', 66, 'a3', '899', 23, 2333][:N]
docs = self.algo.suggest(new_ids, Trials())
# -- assert validity of docs
trials = trials_from_docs(docs)
idxs, vals = miscs_to_idxs_vals(trials.miscs)
assert len(docs) == N
assert len(idxs) == 1
assert len(vals) == 1
print vals
assert idxs['flip'] == new_ids
# -- assert that the random seed matches that of Jan 8/2013
assert np.all(vals['flip'] == [0, 1, 0, 0, 0, 0, 0, 1, 1, 0][:N])
def test_suggest_all(self):
for ii in range(1, 10):
self.setUp()
self.test_suggest_N(ii)
def test_arbitrary_all(self):
for ii in range(1, 10):
self.setUp()
self.test_arbitrary_range(ii)
def test_failure():
#XXX also test the Bandit.exceptions mechanism that actually catches them
class BanditE(Exception):
pass
class DummyBandit(Bandit):
param_gen = {"loss": 10}
def __init__(self):
super(DummyBandit, self).__init__(self.param_gen)
def evaluate(self, config, ctrl):
raise BanditE()
trials = Trials()
bandit_algo = Random(DummyBandit())
exp = Experiment(trials, bandit_algo, async=False)
exp.run(1)
trials.refresh()
assert len(trials) == 0
assert len(trials._dynamic_trials) == 1
assert trials._dynamic_trials[0]['state'] == JOB_STATE_ERROR
assert trials._dynamic_trials[0]['misc']['error'] != None
exp.catch_bandit_exceptions = False
nose.tools.assert_raises(BanditE, exp.run, 1)
trials.refresh()
assert len(trials) == 0
assert len(trials._dynamic_trials) == 2
assert trials._dynamic_trials[1]['state'] == JOB_STATE_ERROR
assert trials._dynamic_trials[1]['misc']['error'] != None
def foo(self):
self.bandit = bandit = Bandit(self.expr)
self.algo = algo = Random(bandit)
if hasattr(self, 'n_randints'):
n_randints = len(filter(
lambda x: x.name == 'randint',
algo.vh.params.values()))
assert n_randints == self.n_randints
self.trials = Trials()
self.experiment = Experiment(self.trials, algo, async=False)
self.experiment.run(5)
self.output = output = []
for trial in self.trials._trials:
print ''
tmp = []
for nid in trial['misc']['idxs']:
thing = (
nid,
trial['misc']['idxs'][nid],
trial['misc']['vals'][nid])
print thing
tmp.append(thing)
tmp.sort()
output.append(tmp)
print repr(output)
print repr(self.wanted)
# -- think of a more robust way to test these things
# or, if the sampling style is to be nailed down,
# put it in and be sure of it.
raise nose.SkipTest()
assert output == self.wanted
class TestRandom(unittest.TestCase):
def setUp(self):
self.bandit = coin_flip()
self.algo = Random(self.bandit)
def test_suggest_1(self):
print 'EXPR', self.bandit.expr
docs = self.algo.suggest([0], Trials())
assert len(docs) == 1
print 'DOCS', docs
# -- assert validity of docs
trials = trials_from_docs(docs)
print 'TRIALS', trials
assert docs[0]['misc']['idxs']['flip'] == [0]
idxs, vals = miscs_to_idxs_vals(trials.miscs)
assert idxs['flip'] == [0]
def test_suggest_5(self):
docs = self.algo.suggest(range(5), Trials())
print docs
assert len(docs) == 5
# -- assert validity of docs
trials = trials_from_docs(docs)
idxs, vals = miscs_to_idxs_vals(trials.miscs)
print idxs
print vals
assert len(idxs) == 1
assert len(vals) == 1
assert idxs['flip'] == range(5)
assert np.all(vals['flip'] == [1, 1, 0, 1, 0])
def test_arbitrary_range(self):
new_ids = [-2, 0, 7, 'a', '007']
docs = self.algo.suggest(new_ids, Trials())
# -- assert validity of docs
trials = trials_from_docs(docs)
idxs, vals = miscs_to_idxs_vals(trials.miscs)
assert len(docs) == 5
assert len(idxs) == 1
assert len(vals) == 1
print vals
assert idxs['flip'] == new_ids
assert np.all(vals['flip'] == [0, 1, 0, 1, 1])
class TestRandom(unittest.TestCase):
def setUp(self):
self.bandit = coin_flip()
self.algo = Random(self.bandit)
def test_suggest_1(self):
print 'EXPR', self.bandit.expr
docs = self.algo.suggest([0], Trials())
assert len(docs) == 1
print 'DOCS', docs
# -- assert validity of docs
trials = trials_from_docs(docs)
print 'TRIALS', trials
assert docs[0]['misc']['idxs']['flip'] == [0]
idxs, vals = miscs_to_idxs_vals(trials.miscs)
assert idxs['flip'] == [0]
def test_suggest_5(self):
docs = self.algo.suggest(range(5), Trials())
print docs
assert len(docs) == 5
# -- assert validity of docs
trials = trials_from_docs(docs)
idxs, vals = miscs_to_idxs_vals(trials.miscs)
print idxs
print vals
assert len(idxs) == 1
assert len(vals) == 1
assert idxs['flip'] == range(5)
assert np.all(vals['flip'] == [1, 1, 0, 1, 0])
def test_arbitrary_range(self):
new_ids = [-2, 0, 7, 'a', '007']
docs = self.algo.suggest(new_ids, Trials())
# -- assert validity of docs
trials = trials_from_docs(docs)
idxs, vals = miscs_to_idxs_vals(trials.miscs)
assert len(docs) == 5
assert len(idxs) == 1
assert len(vals) == 1
print vals
assert idxs['flip'] == new_ids
assert np.all(vals['flip'] == [0, 1, 0, 1, 1])
def setUp(self):
self.bandit = coin_flip()
self.algo = Random(self.bandit)
self.trials = Trials()
self.experiment = Experiment(self.trials, self.algo, async=False)
def setUp(self):
self.bandit = coin_flip()
self.algo = Random(self.bandit)
def setUp(self):
self.bandit = CoinFlip()
self.algo = Random(self.bandit)
