示例#1
0
    def test_seeds_AAB(self):
        # launch 3 simultaneous experiments with seeds A, A, B.
        # Verify all experiments run to completion.
        # Verify first two experiments run identically.
        # Verify third experiment runs differently.

        exp_keys = ['A0', 'A1', 'B']
        seeds = [1, 1, 2]
        n_workers = 2
        jobs_per_thread = 6
        # -- total jobs = 2 * 6 = 12
        # -- divided by 3 experiments: 4 jobs per fmin
        max_evals = (n_workers * jobs_per_thread) // len(exp_keys)

        # -- should not matter which domain is used here
        domain = gauss_wave2()

        cPickle.dumps(domain.expr)
        cPickle.dumps(passthrough)

        worker_threads = [
            threading.Thread(target=TestExperimentWithThreads.worker_thread_fn,
                             args=(('hostname', ii), jobs_per_thread, 30.0))
            for ii in range(n_workers)
        ]

        with TempMongo() as tm:
            mj = tm.mongo_jobs('foodb')
            trials_list = [
                MongoTrials(tm.connection_string('foodb'), key)
                for key in exp_keys
            ]

            fmin_threads = [
                threading.Thread(
                    target=TestExperimentWithThreads.fmin_thread_fn,
                    args=(domain.expr, trials, max_evals, seed))
                for seed, trials in zip(seeds, trials_list)
            ]

            try:
                [th.start() for th in worker_threads + fmin_threads]
            finally:
                print('joining worker threads...')
                [th.join() for th in worker_threads + fmin_threads]

            # -- not using an exp_key gives a handle to all the trials
            #    in foodb
            all_trials = MongoTrials(tm.connection_string('foodb'))
            self.assertEqual(len(all_trials), n_workers * jobs_per_thread)

            # Verify that the fmin calls terminated correctly:
            for trials in trials_list:
                self.assertEqual(trials.count_by_state_synced(JOB_STATE_DONE),
                                 max_evals)
                self.assertEqual(
                    trials.count_by_state_unsynced(JOB_STATE_DONE), max_evals)
                self.assertEqual(len(trials), max_evals)

            # Verify that the first two experiments match.
            # (Do these need sorting by trial id?)
            trials_A0, trials_A1, trials_B0 = trials_list
            self.assertEqual([t['misc']['vals'] for t in trials_A0.trials],
                             [t['misc']['vals'] for t in trials_A1.trials])

            # Verify that the last experiment does not match.
            # (Do these need sorting by trial id?)
            self.assertNotEqual([t['misc']['vals'] for t in trials_A0.trials],
                                [t['misc']['vals'] for t in trials_B0.trials])
示例#2
0
import unittest
from hyperopt.base import Trials, trials_from_docs, miscs_to_idxs_vals
from hyperopt import rand
from hyperopt.tests.test_base import Suggest_API
from test_domains import gauss_wave2, coin_flip

TestRand = Suggest_API.make_tst_class(rand.suggest, gauss_wave2(), 'TestRand')


class TestRand(unittest.TestCase):

    def test_seeding(self):
        # -- assert that the seeding works a particular way

        domain = coin_flip()
        docs = rand.suggest(range(10), domain, Trials(), seed=123)
        trials = trials_from_docs(docs)
        idxs, vals = miscs_to_idxs_vals(trials.miscs)

        # Passes Nov 8 / 2013 
        self.assertEqual(list(idxs['flip']), range(10))
        self.assertEqual(list(vals['flip']), [0, 1, 0, 0, 0, 0, 0, 1, 1, 0])

    # -- TODO: put in a test that guarantees that
    #          stochastic nodes are sampled in a paricular order.
示例#3
0
import unittest
from hyperopt.base import Trials, trials_from_docs, miscs_to_idxs_vals
from hyperopt import rand
from hyperopt.tests.test_base import Suggest_API
from test_domains import gauss_wave2, coin_flip

TestRand = Suggest_API.make_tst_class(rand.suggest, gauss_wave2(), 'TestRand')


class TestRand(unittest.TestCase):
    def test_seeding(self):
        # -- assert that the seeding works a particular way

        domain = coin_flip()
        docs = rand.suggest(range(10), domain, Trials(), seed=123)
        trials = trials_from_docs(docs)
        idxs, vals = miscs_to_idxs_vals(trials.miscs)

        # Passes Nov 8 / 2013
        self.assertEqual(list(idxs['flip']), range(10))
        self.assertEqual(list(vals['flip']), [0, 1, 0, 0, 0, 0, 0, 1, 1, 0])

    # -- TODO: put in a test that guarantees that
    #          stochastic nodes are sampled in a paricular order.
示例#4
0
    def test_seeds_AAB(self):
        # launch 3 simultaneous experiments with seeds A, A, B.
        # Verify all experiments run to completion.
        # Verify first two experiments run identically.
        # Verify third experiment runs differently.

        exp_keys = ['A0', 'A1', 'B']
        seeds = [1, 1, 2]
        n_workers = 2
        jobs_per_thread = 6
        # -- total jobs = 2 * 6 = 12
        # -- divided by 3 experiments: 4 jobs per fmin
        max_evals = (n_workers * jobs_per_thread) // len(exp_keys)

        # -- should not matter which domain is used here
        domain = gauss_wave2()

        cPickle.dumps(domain.expr)
        cPickle.dumps(passthrough)


        worker_threads = [
            threading.Thread(
                target=TestExperimentWithThreads.worker_thread_fn,
                args=(('hostname', ii), jobs_per_thread, 30.0))
            for ii in range(n_workers)]

        with TempMongo() as tm:
            mj = tm.mongo_jobs('foodb')
            trials_list = [
                MongoTrials(tm.connection_string('foodb'), key)
                for key in exp_keys]

            fmin_threads = [
                threading.Thread(
                    target=TestExperimentWithThreads.fmin_thread_fn,
                    args=(domain.expr, trials, max_evals, seed))
                for seed, trials in zip(seeds, trials_list)]

            try:
                [th.start() for th in worker_threads + fmin_threads]
            finally:
                print('joining worker threads...')
                [th.join() for th in worker_threads + fmin_threads]

            # -- not using an exp_key gives a handle to all the trials
            #    in foodb
            all_trials = MongoTrials(tm.connection_string('foodb'))
            self.assertEqual(len(all_trials), n_workers * jobs_per_thread)

            # Verify that the fmin calls terminated correctly:
            for trials in trials_list:
                self.assertEqual(
                    trials.count_by_state_synced(JOB_STATE_DONE),
                    max_evals)
                self.assertEqual(
                    trials.count_by_state_unsynced(JOB_STATE_DONE),
                    max_evals)
                self.assertEqual(len(trials), max_evals)


            # Verify that the first two experiments match.
            # (Do these need sorting by trial id?)
            trials_A0, trials_A1, trials_B0 = trials_list
            self.assertEqual(
                [t['misc']['vals'] for t in trials_A0.trials],
                [t['misc']['vals'] for t in trials_A1.trials])

            # Verify that the last experiment does not match.
            # (Do these need sorting by trial id?)
            self.assertNotEqual(
                [t['misc']['vals'] for t in trials_A0.trials],
                [t['misc']['vals'] for t in trials_B0.trials])