def test_member_ready(keras_model):
"""This member should be ready after the sixth and twelth epochs."""
from pybt.policy.ready import ReadyAfter
from pybt.policy.done import StopAfter
m = Member(SimpleWrapper(keras_model),
1,
stopping_criteria=StopAfter(epochs=13),
ready_strategy=ReadyAfter(epochs=6),
step_args={
'epochs_per_step': 3,
'fit_args': {
'x': {},
'y': {}
}
},
eval_args={})
assert m.ready() == False
m.step()
assert m.ready() == False
m.step()
assert m.ready() == True
m.step()
assert m.ready() == False
m.step()
assert m.ready() == True
m.step()
assert m.ready() == False
def test_quadratic_unit_steps():
"""Test that we can train unit-sized steps."""
import random
random.seed(0)
stop_after_epochs = 10
ready_after_epochs = 1
epochs_per_step = 1
t = Trainer(SimpleWrapper(QuadraticModel()),
step_args={
'epochs_per_step': epochs_per_step,
'fit_args': {
'x': {},
'y': {}
}
},
eval_args={
'x': {},
'y': {}
},
stopping_criteria=StopAfter(stop_after_epochs),
ready_strategy=ReadyAfter(ready_after_epochs))
model, score = t.train()
assert floor(score) == 54
loss, acc = model.evaluate({})
def test_linear_small_steps_when_ready():
"""Test that we can train multi-epoch steps."""
stop_after_epochs = 30
ready_after_epochs = 6
epochs_per_step = 3
t = Trainer(SimpleWrapper(LinearModel()),
step_args={
'epochs_per_step': epochs_per_step,
'fit_args': {
'x': {},
'y': {}
}
},
eval_args={
'x': {},
'y': {}
},
stopping_criteria=StopAfter(stop_after_epochs),
ready_strategy=ReadyAfter(ready_after_epochs))
model, score = t.train()
assert score == 30
assert model.evaluate({}) == (-30, 30)
assert len(t.observations()) == (stop_after_epochs // epochs_per_step) + 1
def test_multiple_train_calls():
stop_after_epochs = 30
ready_after_epochs = 1
epochs_per_step = 3
t = Trainer(SimpleWrapper(LinearModel()),
step_args={
'epochs_per_step': epochs_per_step,
'fit_args': {
'x': {},
'y': {}
}
},
eval_args={
'x': {},
'y': {}
},
stopping_criteria=StopAfter(stop_after_epochs),
ready_strategy=ReadyAfter(ready_after_epochs))
model, score = t.train()
assert score == 30
assert model.evaluate({}) == (-30, 30)
assert len(t.observations()) == (stop_after_epochs // epochs_per_step) + 1
# Nothing should happen because the done criteria has already been met.
m2, s2 = t.train()
assert s2 == 30
def test_linear_unit_steps(linear_model):
"""Test that we can train unit-sized steps."""
stop_after_epochs = 10
ready_after_epochs = 1
epochs_per_step = 1
t = Trainer(linear_model,
step_args={
'epochs_per_step': epochs_per_step,
'fit_args': {
'x': {},
'y': {}
}
},
eval_args={
'x': {},
'y': {}
},
stopping_criteria=StopAfter(stop_after_epochs),
ready_strategy=ReadyAfter(ready_after_epochs))
model, score = t.train()
assert score == 10
assert model.evaluate({}) == (-10, 10)
assert len(t.observations()) == (stop_after_epochs // epochs_per_step) + 1
def __init__(self, model, stopping_criteria=StopAfter(10),
ready_strategy=ReadyAfter(2), exploit_strategy=Truncation(),
step_args={}, eval_args={}):
logger.debug('Trainer(model={}, stopping_criteria={}, '
'ready_strategy={}, exploit_strategy={}, step_args={}, '
'eval_args={})'.format(model, stopping_criteria, ready_strategy,
exploit_strategy, step_args, eval_args))
self._exploit_strategy = exploit_strategy
member = Member(model, 0, stopping_criteria,
ready_strategy, step_args, eval_args)
self._members = Population(member)
def test_member_done(keras_model):
from pybt.policy.done import StopAfter
m = Member(SimpleWrapper(keras_model),
1,
step_args={
'epochs_per_step': 2,
'fit_args': {
'x': {},
'y': {}
}
},
eval_args={},
stopping_criteria=StopAfter(epochs=4))
assert m.done() == False
m.step()
assert m.done() == False
m.step()
assert m.done() == True
def test_truncation(keras_model):
"""After 5 steps, we should swap the current member with one in the
top 25%."""
from pybt.policy.exploit import Truncation
from pybt.policy.ready import ReadyAfter
from pybt.policy.done import StopAfter
t = Trainer(model=SimpleWrapper(keras_model),
stopping_criteria=StopAfter(5),
ready_strategy=ReadyAfter(5),
exploit_strategy=Truncation(upper=.4, lower=.4),
step_args={
'epochs_per_step': 1,
'fit_args': {
'x': {},
'y': {}
}
})
_, score = t.train()
assert score in [84., 90.]
model = models.Sequential()
model.add(Dense(16, activation='relu', input_shape=(10000, )))
model.add(Dense(16, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer=RMSprop(lr=0.001),
loss=binary_crossentropy,
metrics=['accuracy'])
# Create a population with this model and train
m = KerasModelWrapper(model,
optimizer=RMSprop(lr=0.001),
loss=binary_crossentropy,
metrics=['accuracy'])
t = Trainer(model=m,
stopping_criteria=StopAfter(4),
ready_strategy=ReadyAfter(1),
step_args={
'epochs_per_step': 1,
'fit_args': {
'x': partial_x_train,
'y': partial_y_train
}
},
eval_args={
'x': x_val,
'y': y_val
})
model, score = t.train()
print(t)