def test_algorithm_is_done(monkeypatch, producer):
"""Verify that producer won't register new samples if algorithm is done meanwhile."""
producer.experiment.max_trials = 8
producer.experiment.algorithms.algorithm.max_trials = 8
producer.experiment.pool_size = 10
producer = Producer(producer.experiment)
def suggest_one_only(self, num=1):
"""Return only one point, whatever `num` is"""
return [("gru", "rnn")]
monkeypatch.delattr(producer.experiment.algorithms.algorithm.__class__,
"is_done")
monkeypatch.setattr(producer.experiment.algorithms.algorithm.__class__,
"suggest", suggest_one_only)
assert producer.experiment.pool_size == 10
trials_in_exp_before = len(producer.experiment.fetch_trials())
assert trials_in_exp_before == producer.experiment.max_trials - 1
producer.update()
producer.produce()
assert len(
producer.experiment.fetch_trials()) == producer.experiment.max_trials
assert producer.naive_algorithm.is_done
assert not producer.experiment.is_done
def test_concurent_producers(producer, storage, random_dt):
"""Test concurrent production of new trials."""
trials_in_db_before = len(storage._fetch_trials({}))
new_trials_in_db_before = len(storage._fetch_trials({"status": "new"}))
# Set so that first producer's algorithm generate valid point on first time
# And second producer produce same point and thus must produce next one two.
# Hence, we know that producer algo will have _num == 1 and
# second producer algo will have _num == 2
producer.algorithm.algorithm.possible_values = [("gru", "rnn"),
("gru", "gru")]
# Make sure it starts from index 0
producer.algorithm.seed_rng(0)
assert producer.experiment.pool_size == 1
second_producer = Producer(producer.experiment)
second_producer.algorithm = copy.deepcopy(producer.algorithm)
producer.update()
second_producer.update()
producer.produce()
second_producer.produce()
# Algorithm was required to suggest some trials
num_new_points = producer.algorithm.algorithm._num
assert num_new_points == 1 # pool size
num_new_points = second_producer.algorithm.algorithm._num
assert num_new_points == 2 # pool size
# `num_new_points` new trials were registered at database
assert len(storage._fetch_trials({})) == trials_in_db_before + 2
assert len(storage._fetch_trials({"status":
"new"})) == new_trials_in_db_before + 2
new_trials = list(
storage._fetch_trials({
"status": "new",
"submit_time": random_dt
}))
assert new_trials[0].experiment == producer.experiment.id
assert new_trials[0].start_time is None
assert new_trials[0].end_time is None
assert new_trials[0].results == []
assert new_trials[0].params == {
"/decoding_layer": "gru",
"/encoding_layer": "rnn",
}
assert new_trials[1].params == {
"/decoding_layer": "gru",
"/encoding_layer": "gru",
}
def workon(experiment):
"""Try to find solution to the search problem defined in `experiment`."""
producer = Producer(experiment)
consumer = Consumer(experiment)
log.debug("##### Init Experiment #####")
while True:
log.debug("#### Try to reserve a new trial to evaluate.")
trial = experiment.reserve_trial(score_handle=producer.algorithm.score)
if trial is None:
log.debug("#### Failed to pull a new trial from database.")
log.debug(
"#### Fetch most recent completed trials and update algorithm."
)
producer.update()
log.debug("#### Poll for experiment termination.")
if experiment.is_done:
break
log.debug("#### Produce new trials.")
producer.produce()
else:
log.debug(
"#### Successfully reserved %s to evaluate. Consuming...",
trial)
consumer.consume(trial)
stats = experiment.stats
best = Database().read('trials', {'_id': stats['best_trials_id']})[0]
stats_stream = io.StringIO()
pprint.pprint(stats, stream=stats_stream)
stats_string = stats_stream.getvalue()
best_stream = io.StringIO()
pprint.pprint(best['params'], stream=best_stream)
best_string = best_stream.getvalue()
log.info("##### Search finished successfully #####")
log.info("\nRESULTS\n=======\n%s\n", stats_string)
log.info("\nBEST PARAMETERS\n===============\n%s", best_string)
def test_concurent_producers(producer, database, random_dt):
"""Test concurrent production of new trials."""
trials_in_db_before = database.trials.count()
new_trials_in_db_before = database.trials.count({'status': 'new'})
print(producer.experiment.fetch_trials({}))
# Set so that first producer's algorithm generate valid point on first time
# And second producer produce same point and thus must produce next one two.
# Hence, we know that producer algo will have _num == 1 and
# second producer algo will have _num == 2
producer.algorithm.algorithm.possible_values = [('rnn', 'gru'),
('gru', 'gru')]
# Make sure it starts from index 0
producer.algorithm.seed_rng(0)
assert producer.experiment.pool_size == 1
second_producer = Producer(producer.experiment)
second_producer.algorithm = copy.deepcopy(producer.algorithm)
producer.update()
second_producer.update()
print(producer.algorithm.algorithm._index)
print(second_producer.algorithm.algorithm._index)
producer.produce()
print(producer.algorithm.algorithm._index)
print(second_producer.algorithm.algorithm._index)
second_producer.produce()
# Algorithm was required to suggest some trials
num_new_points = producer.algorithm.algorithm._num
assert num_new_points == 1 # pool size
num_new_points = second_producer.algorithm.algorithm._num
assert num_new_points == 2 # pool size
# `num_new_points` new trials were registered at database
assert database.trials.count() == trials_in_db_before + 2
assert database.trials.count({'status':
'new'}) == new_trials_in_db_before + 2
new_trials = list(
database.trials.find({
'status': 'new',
'submit_time': random_dt
}))
assert new_trials[0]['experiment'] == producer.experiment.name
assert new_trials[0]['start_time'] is None
assert new_trials[0]['end_time'] is None
assert new_trials[0]['results'] == []
assert new_trials[0]['params'] == [{
'name': '/encoding_layer',
'type': 'categorical',
'value': 'rnn'
}, {
'name': '/decoding_layer',
'type': 'categorical',
'value': 'gru'
}]
assert new_trials[1]['params'] == [{
'name': '/encoding_layer',
'type': 'categorical',
'value': 'gru'
}, {
'name': '/decoding_layer',
'type': 'categorical',
'value': 'gru'
}]
def test_concurent_producers(producer, database, random_dt):
"""Test concurrent production of new trials."""
trials_in_db_before = database.trials.count()
new_trials_in_db_before = database.trials.count({"status": "new"})
# Set so that first producer's algorithm generate valid point on first time
# And second producer produce same point and thus must produce next one two.
# Hence, we know that producer algo will have _num == 1 and
# second producer algo will have _num == 2
producer.algorithm.algorithm.possible_values = [("gru", "rnn"),
("gru", "gru")]
# Make sure it starts from index 0
producer.algorithm.seed_rng(0)
assert producer.experiment.pool_size == 1
second_producer = Producer(producer.experiment)
second_producer.algorithm = copy.deepcopy(producer.algorithm)
producer.update()
second_producer.update()
producer.produce()
second_producer.produce()
# Algorithm was required to suggest some trials
num_new_points = producer.algorithm.algorithm._num
assert num_new_points == 1 # pool size
num_new_points = second_producer.algorithm.algorithm._num
assert num_new_points == 2 # pool size
# `num_new_points` new trials were registered at database
assert database.trials.count() == trials_in_db_before + 2
assert database.trials.count({"status":
"new"}) == new_trials_in_db_before + 2
new_trials = list(
database.trials.find({
"status": "new",
"submit_time": random_dt
}))
assert new_trials[0]["experiment"] == producer.experiment.name
assert new_trials[0]["start_time"] is None
assert new_trials[0]["end_time"] is None
assert new_trials[0]["results"] == []
assert new_trials[0]["params"] == [
{
"name": "/decoding_layer",
"type": "categorical",
"value": "gru"
},
{
"name": "/encoding_layer",
"type": "categorical",
"value": "rnn"
},
]
assert new_trials[1]["params"] == [
{
"name": "/decoding_layer",
"type": "categorical",
"value": "gru"
},
{
"name": "/encoding_layer",
"type": "categorical",
"value": "gru"
},
]