예제 #1
0
def test_algorithm_is_done(monkeypatch, producer):
    """Verify that producer won't register new samples if algorithm is done meanwhile."""
    producer.experiment.max_trials = 8
    producer.experiment.algorithms.algorithm.max_trials = 8
    producer.experiment.pool_size = 10
    producer = Producer(producer.experiment)

    def suggest_one_only(self, num=1):
        """Return only one point, whatever `num` is"""
        return [("gru", "rnn")]

    monkeypatch.delattr(producer.experiment.algorithms.algorithm.__class__,
                        "is_done")
    monkeypatch.setattr(producer.experiment.algorithms.algorithm.__class__,
                        "suggest", suggest_one_only)

    assert producer.experiment.pool_size == 10
    trials_in_exp_before = len(producer.experiment.fetch_trials())
    assert trials_in_exp_before == producer.experiment.max_trials - 1

    producer.update()
    producer.produce()

    assert len(
        producer.experiment.fetch_trials()) == producer.experiment.max_trials
    assert producer.naive_algorithm.is_done
    assert not producer.experiment.is_done
예제 #2
0
def test_concurent_producers(producer, storage, random_dt):
    """Test concurrent production of new trials."""
    trials_in_db_before = len(storage._fetch_trials({}))
    new_trials_in_db_before = len(storage._fetch_trials({"status": "new"}))

    # Set so that first producer's algorithm generate valid point on first time
    # And second producer produce same point and thus must produce next one two.
    # Hence, we know that producer algo will have _num == 1 and
    # second producer algo will have _num == 2
    producer.algorithm.algorithm.possible_values = [("gru", "rnn"),
                                                    ("gru", "gru")]
    # Make sure it starts from index 0
    producer.algorithm.seed_rng(0)

    assert producer.experiment.pool_size == 1

    second_producer = Producer(producer.experiment)
    second_producer.algorithm = copy.deepcopy(producer.algorithm)

    producer.update()
    second_producer.update()

    producer.produce()
    second_producer.produce()

    # Algorithm was required to suggest some trials
    num_new_points = producer.algorithm.algorithm._num
    assert num_new_points == 1  # pool size
    num_new_points = second_producer.algorithm.algorithm._num
    assert num_new_points == 2  # pool size

    # `num_new_points` new trials were registered at database
    assert len(storage._fetch_trials({})) == trials_in_db_before + 2
    assert len(storage._fetch_trials({"status":
                                      "new"})) == new_trials_in_db_before + 2
    new_trials = list(
        storage._fetch_trials({
            "status": "new",
            "submit_time": random_dt
        }))
    assert new_trials[0].experiment == producer.experiment.id
    assert new_trials[0].start_time is None
    assert new_trials[0].end_time is None
    assert new_trials[0].results == []
    assert new_trials[0].params == {
        "/decoding_layer": "gru",
        "/encoding_layer": "rnn",
    }

    assert new_trials[1].params == {
        "/decoding_layer": "gru",
        "/encoding_layer": "gru",
    }
예제 #3
0
def workon(experiment):
    """Try to find solution to the search problem defined in `experiment`."""
    producer = Producer(experiment)
    consumer = Consumer(experiment)

    log.debug("#####  Init Experiment  #####")
    while True:
        log.debug("#### Try to reserve a new trial to evaluate.")
        trial = experiment.reserve_trial(score_handle=producer.algorithm.score)

        if trial is None:
            log.debug("#### Failed to pull a new trial from database.")

            log.debug(
                "#### Fetch most recent completed trials and update algorithm."
            )
            producer.update()

            log.debug("#### Poll for experiment termination.")
            if experiment.is_done:
                break

            log.debug("#### Produce new trials.")
            producer.produce()

        else:
            log.debug(
                "#### Successfully reserved %s to evaluate. Consuming...",
                trial)
            consumer.consume(trial)

    stats = experiment.stats
    best = Database().read('trials', {'_id': stats['best_trials_id']})[0]

    stats_stream = io.StringIO()
    pprint.pprint(stats, stream=stats_stream)
    stats_string = stats_stream.getvalue()

    best_stream = io.StringIO()
    pprint.pprint(best['params'], stream=best_stream)
    best_string = best_stream.getvalue()

    log.info("#####  Search finished successfully  #####")
    log.info("\nRESULTS\n=======\n%s\n", stats_string)
    log.info("\nBEST PARAMETERS\n===============\n%s", best_string)
예제 #4
0
def test_concurent_producers(producer, database, random_dt):
    """Test concurrent production of new trials."""
    trials_in_db_before = database.trials.count()
    new_trials_in_db_before = database.trials.count({'status': 'new'})

    print(producer.experiment.fetch_trials({}))

    # Set so that first producer's algorithm generate valid point on first time
    # And second producer produce same point and thus must produce next one two.
    # Hence, we know that producer algo will have _num == 1 and
    # second producer algo will have _num == 2
    producer.algorithm.algorithm.possible_values = [('rnn', 'gru'),
                                                    ('gru', 'gru')]
    # Make sure it starts from index 0
    producer.algorithm.seed_rng(0)

    assert producer.experiment.pool_size == 1

    second_producer = Producer(producer.experiment)
    second_producer.algorithm = copy.deepcopy(producer.algorithm)

    producer.update()
    second_producer.update()

    print(producer.algorithm.algorithm._index)
    print(second_producer.algorithm.algorithm._index)
    producer.produce()
    print(producer.algorithm.algorithm._index)
    print(second_producer.algorithm.algorithm._index)
    second_producer.produce()

    # Algorithm was required to suggest some trials
    num_new_points = producer.algorithm.algorithm._num
    assert num_new_points == 1  # pool size
    num_new_points = second_producer.algorithm.algorithm._num
    assert num_new_points == 2  # pool size

    # `num_new_points` new trials were registered at database
    assert database.trials.count() == trials_in_db_before + 2
    assert database.trials.count({'status':
                                  'new'}) == new_trials_in_db_before + 2
    new_trials = list(
        database.trials.find({
            'status': 'new',
            'submit_time': random_dt
        }))
    assert new_trials[0]['experiment'] == producer.experiment.name
    assert new_trials[0]['start_time'] is None
    assert new_trials[0]['end_time'] is None
    assert new_trials[0]['results'] == []
    assert new_trials[0]['params'] == [{
        'name': '/encoding_layer',
        'type': 'categorical',
        'value': 'rnn'
    }, {
        'name': '/decoding_layer',
        'type': 'categorical',
        'value': 'gru'
    }]

    assert new_trials[1]['params'] == [{
        'name': '/encoding_layer',
        'type': 'categorical',
        'value': 'gru'
    }, {
        'name': '/decoding_layer',
        'type': 'categorical',
        'value': 'gru'
    }]
예제 #5
0
def test_concurent_producers(producer, database, random_dt):
    """Test concurrent production of new trials."""
    trials_in_db_before = database.trials.count()
    new_trials_in_db_before = database.trials.count({"status": "new"})

    # Set so that first producer's algorithm generate valid point on first time
    # And second producer produce same point and thus must produce next one two.
    # Hence, we know that producer algo will have _num == 1 and
    # second producer algo will have _num == 2
    producer.algorithm.algorithm.possible_values = [("gru", "rnn"),
                                                    ("gru", "gru")]
    # Make sure it starts from index 0
    producer.algorithm.seed_rng(0)

    assert producer.experiment.pool_size == 1

    second_producer = Producer(producer.experiment)
    second_producer.algorithm = copy.deepcopy(producer.algorithm)

    producer.update()
    second_producer.update()

    producer.produce()
    second_producer.produce()

    # Algorithm was required to suggest some trials
    num_new_points = producer.algorithm.algorithm._num
    assert num_new_points == 1  # pool size
    num_new_points = second_producer.algorithm.algorithm._num
    assert num_new_points == 2  # pool size

    # `num_new_points` new trials were registered at database
    assert database.trials.count() == trials_in_db_before + 2
    assert database.trials.count({"status":
                                  "new"}) == new_trials_in_db_before + 2
    new_trials = list(
        database.trials.find({
            "status": "new",
            "submit_time": random_dt
        }))
    assert new_trials[0]["experiment"] == producer.experiment.name
    assert new_trials[0]["start_time"] is None
    assert new_trials[0]["end_time"] is None
    assert new_trials[0]["results"] == []
    assert new_trials[0]["params"] == [
        {
            "name": "/decoding_layer",
            "type": "categorical",
            "value": "gru"
        },
        {
            "name": "/encoding_layer",
            "type": "categorical",
            "value": "rnn"
        },
    ]

    assert new_trials[1]["params"] == [
        {
            "name": "/decoding_layer",
            "type": "categorical",
            "value": "gru"
        },
        {
            "name": "/encoding_layer",
            "type": "categorical",
            "value": "gru"
        },
    ]