def test_seeds_AAB(self): # launch 3 simultaneous experiments with seeds A, A, B. # Verify all experiments run to completion. # Verify first two experiments run identically. # Verify third experiment runs differently. exp_keys = ['A0', 'A1', 'B'] seeds = [1, 1, 2] n_workers = 2 jobs_per_thread = 6 # -- total jobs = 2 * 6 = 12 # -- divided by 3 experiments: 4 jobs per fmin max_evals = (n_workers * jobs_per_thread) // len(exp_keys) # -- should not matter which domain is used here domain = gauss_wave2() cPickle.dumps(domain.expr) cPickle.dumps(passthrough) worker_threads = [ threading.Thread(target=TestExperimentWithThreads.worker_thread_fn, args=(('hostname', ii), jobs_per_thread, 30.0)) for ii in range(n_workers) ] with TempMongo() as tm: mj = tm.mongo_jobs('foodb') trials_list = [ MongoTrials(tm.connection_string('foodb'), key) for key in exp_keys ] fmin_threads = [ threading.Thread( target=TestExperimentWithThreads.fmin_thread_fn, args=(domain.expr, trials, max_evals, seed)) for seed, trials in zip(seeds, trials_list) ] try: [th.start() for th in worker_threads + fmin_threads] finally: print('joining worker threads...') [th.join() for th in worker_threads + fmin_threads] # -- not using an exp_key gives a handle to all the trials # in foodb all_trials = MongoTrials(tm.connection_string('foodb')) self.assertEqual(len(all_trials), n_workers * jobs_per_thread) # Verify that the fmin calls terminated correctly: for trials in trials_list: self.assertEqual(trials.count_by_state_synced(JOB_STATE_DONE), max_evals) self.assertEqual( trials.count_by_state_unsynced(JOB_STATE_DONE), max_evals) self.assertEqual(len(trials), max_evals) # Verify that the first two experiments match. # (Do these need sorting by trial id?) trials_A0, trials_A1, trials_B0 = trials_list self.assertEqual([t['misc']['vals'] for t in trials_A0.trials], [t['misc']['vals'] for t in trials_A1.trials]) # Verify that the last experiment does not match. # (Do these need sorting by trial id?) self.assertNotEqual([t['misc']['vals'] for t in trials_A0.trials], [t['misc']['vals'] for t in trials_B0.trials])
import unittest from hyperopt.base import Trials, trials_from_docs, miscs_to_idxs_vals from hyperopt import rand from hyperopt.tests.test_base import Suggest_API from test_domains import gauss_wave2, coin_flip TestRand = Suggest_API.make_tst_class(rand.suggest, gauss_wave2(), 'TestRand') class TestRand(unittest.TestCase): def test_seeding(self): # -- assert that the seeding works a particular way domain = coin_flip() docs = rand.suggest(range(10), domain, Trials(), seed=123) trials = trials_from_docs(docs) idxs, vals = miscs_to_idxs_vals(trials.miscs) # Passes Nov 8 / 2013 self.assertEqual(list(idxs['flip']), range(10)) self.assertEqual(list(vals['flip']), [0, 1, 0, 0, 0, 0, 0, 1, 1, 0]) # -- TODO: put in a test that guarantees that # stochastic nodes are sampled in a paricular order.
def test_seeds_AAB(self): # launch 3 simultaneous experiments with seeds A, A, B. # Verify all experiments run to completion. # Verify first two experiments run identically. # Verify third experiment runs differently. exp_keys = ['A0', 'A1', 'B'] seeds = [1, 1, 2] n_workers = 2 jobs_per_thread = 6 # -- total jobs = 2 * 6 = 12 # -- divided by 3 experiments: 4 jobs per fmin max_evals = (n_workers * jobs_per_thread) // len(exp_keys) # -- should not matter which domain is used here domain = gauss_wave2() cPickle.dumps(domain.expr) cPickle.dumps(passthrough) worker_threads = [ threading.Thread( target=TestExperimentWithThreads.worker_thread_fn, args=(('hostname', ii), jobs_per_thread, 30.0)) for ii in range(n_workers)] with TempMongo() as tm: mj = tm.mongo_jobs('foodb') trials_list = [ MongoTrials(tm.connection_string('foodb'), key) for key in exp_keys] fmin_threads = [ threading.Thread( target=TestExperimentWithThreads.fmin_thread_fn, args=(domain.expr, trials, max_evals, seed)) for seed, trials in zip(seeds, trials_list)] try: [th.start() for th in worker_threads + fmin_threads] finally: print('joining worker threads...') [th.join() for th in worker_threads + fmin_threads] # -- not using an exp_key gives a handle to all the trials # in foodb all_trials = MongoTrials(tm.connection_string('foodb')) self.assertEqual(len(all_trials), n_workers * jobs_per_thread) # Verify that the fmin calls terminated correctly: for trials in trials_list: self.assertEqual( trials.count_by_state_synced(JOB_STATE_DONE), max_evals) self.assertEqual( trials.count_by_state_unsynced(JOB_STATE_DONE), max_evals) self.assertEqual(len(trials), max_evals) # Verify that the first two experiments match. # (Do these need sorting by trial id?) trials_A0, trials_A1, trials_B0 = trials_list self.assertEqual( [t['misc']['vals'] for t in trials_A0.trials], [t['misc']['vals'] for t in trials_A1.trials]) # Verify that the last experiment does not match. # (Do these need sorting by trial id?) self.assertNotEqual( [t['misc']['vals'] for t in trials_A0.trials], [t['misc']['vals'] for t in trials_B0.trials])