def test_finished_relative(self):
tp = base.TrainingParameters()
start = base.TrainingParameters.DEFAULT_ABSOLUTE * 1000
step = start * base.TrainingParameters.DEFAULT_RELATIVE * 0.9
# Less than
losses = tp.losses()
for i in range(tp.window()):
losses.append(start - (step * i))
self.assertEqual(tp.finished(1, losses),
(True, "relative convergence"), losses)
# Not less than
losses = tp.losses()
for i in range(tp.window()):
losses.append(start - (step * i * 2))
self.assertEqual(tp.finished(1, losses), (False, None), losses)
# Not decreasing
losses = tp.losses()
for i in range(tp.window()):
losses.append(start)
self.assertEqual(tp.finished(1, losses), (False, None), losses)
def test_finished_absolute(self):
tp = base.TrainingParameters()
start = base.TrainingParameters.DEFAULT_ABSOLUTE
step = base.TrainingParameters.DEFAULT_ABSOLUTE / float(tp.window())
# Less than
losses = tp.losses()
for i in range(tp.window()):
losses.append(start - (step * i))
self.assertEqual(tp.finished(1, losses),
(True, "absolute convergence"), losses)
# Not less than
losses = tp.losses()
for i in range(tp.window()):
losses.append((start * 2) - (step * i))
self.assertEqual(tp.finished(1, losses), (False, None), losses)
# Not decreasing
losses = tp.losses()
for i in range(tp.window()):
losses.append(start)
self.assertEqual(tp.finished(1, losses), (False, None), losses)
def test_finished_degrading_recovery(self):
tp = base.TrainingParameters()
start = tp.absolute()
step = start * tp.degradation() * 2.1
step_down = (start * tp.degradation() * 1.1) / (tp.window() * 0.25)
# Degradation - but with recovery
losses = tp.losses()
j = 0
for i in range(tp.window()):
if i >= int(tp.window() * 0.75):
j += 1
losses.append(start + (step - (step_down * j) if i != 0 else 0.0))
self.assertEqual(tp.finished(1, losses), (False, None), losses)
# Degradation - but with insufficient recovery
losses = tp.losses()
j = 0
for i in range(tp.window()):
if i >= int(tp.window() * 0.75) and j == 0:
j += 1
losses.append(start + (step - (step_down * j) if i != 0 else 0.0))
self.assertEqual(tp.finished(1, losses), (True, "degradation"), losses)
def test_non_convergence_relative(self):
tp = base.TrainingParameters().convergence(False)
start = base.TrainingParameters.DEFAULT_ABSOLUTE * 1000
step = start * base.TrainingParameters.DEFAULT_RELATIVE * 0.9
# Less than - but without convergence
losses = tp.losses()
for i in range(tp.window()):
losses.append(start - (step * i))
self.assertEqual(tp.finished(1, losses), (False, None), losses)
def test_non_convergence_absolute(self):
tp = base.TrainingParameters().convergence(False)
start = base.TrainingParameters.DEFAULT_ABSOLUTE
step = base.TrainingParameters.DEFAULT_ABSOLUTE / float(tp.window())
# Less than - but without convergence
losses = tp.losses()
for i in range(tp.window()):
losses.append(start - (step * i))
self.assertEqual(tp.finished(1, losses), (False, None), losses)
def test_finished_epochs(self):
tp = base.TrainingParameters()
losses = tp.losses()
self.assertEqual(
tp.finished(base.TrainingParameters.EPOCHS_DEFAULT + 1, losses),
(True, "maximum epochs"), losses)
self.assertEqual(
tp.finished(base.TrainingParameters.EPOCHS_DEFAULT + 0, losses),
(True, "maximum epochs"), losses)
self.assertEqual(
tp.finished(base.TrainingParameters.EPOCHS_DEFAULT - 1, losses),
(True, "maximum epochs"), losses)
self.assertEqual(
tp.finished(base.TrainingParameters.EPOCHS_DEFAULT - 2, losses),
(False, None), losses)
def test_finished_degrading(self):
tp = base.TrainingParameters()
start = base.TrainingParameters.DEFAULT_ABSOLUTE
step = start * base.TrainingParameters.DEFAULT_DEGRADATION * 1.1
# Degradation
losses = tp.losses()
for i in range(tp.window()):
losses.append(start + (step if i != 0 else 0.0))
self.assertEqual(tp.finished(1, losses), (True, "degradation"), losses)
# No degradation
losses = tp.losses()
for i in range(tp.window()):
losses.append(start + (step if (i + 1) % 2 == 0 else 0.0))
self.assertEqual(tp.finished(1, losses), (False, None), losses)
def test(self, xy_sequences, debug=False, score=False):
assert len(xy_sequences) > 0
training_parameters = mlbase.TrainingParameters() \
.dropout_rate(0)
total_loss = 0.0
total_score = 0.0
case_slot_length = len(str(len(xy_sequences)))
offset = 0
while offset < len(xy_sequences):
batch = xy_sequences[offset:offset + 32]
offset += 32
feed = self.get_training_feed(batch, training_parameters)
time_distributions, loss = self.session.run([self.output_distributions, self.cost], feed_dict=feed)
total_loss += loss
if score:
total_score += self.score(batch, feed, time_distributions, debug, case_slot_length)
if score:
logging.debug("total score for %d instances: %f" % (len(xy_sequences), total_score / len(xy_sequences)))
return -math.exp(total_loss / len(xy_sequences))
mlbase.Xy(("outputs", 1, [.1, .2, .3, .4, .5]), {
"abc": .1,
"def": .6,
"ghi": .3
}),
#mlbase.Xy(("outputs", 1, [.1, .2, .3, .4, .5]), {"abc": .3, "def": .3, "ghi": .4}),
#mlbase.Xy(("outputs", 1, [.5, .4, .3, .2, .1]), {"abc": .4, "def": .4, "ghi": .2}),
#mlbase.Xy(("cells", 0, [.1, .2, .3, .4, .5]), {"abc": .2, "def": .4, "ghi": .4}),
#mlbase.Xy(("cells", 0, [.5, .4, .3, .2, .1]), {"abc": .6, "def": .2, "ghi": .2}),
#mlbase.Xy(("cells", 1, [.1, .2, .3, .4, .5]), {"abc": .35, "def": .35, "ghi": .3}),
#mlbase.Xy(("cells", 1, [.5, .4, .3, .2, .1]), {"abc": .3, "def": .3, "ghi": .4}),
]
loss = predictor.train(
data,
mlbase.TrainingParameters().epochs(2000).relative(0.000005).absolute(
0.5).batch(1))
print("loss: %s" % loss)
for xy in data:
inferred = predictor.evaluate(xy.x).distribution
print("%s:\n expected: %s\n actual: %s" %
(xy.x, adjutant.dict_as_str(xy.y, use_key=False),
adjutant.dict_as_str(inferred, use_key=False)))
print("inference")
test_data = [
("outputs", 0, [.1, .2, .3, .4, .5]),
("outputs", 0, [.2, .2, .3, .4, .5]),
("outputs", 0, [.2, .2, .3, .4, .5]),
#("cells", 1, [.1, .2, .3, .3, .5]),
#("cells", 1, [.1, .2, .2, .3, .5]),
def converging_train(self,
data_streams,
model_dir,
batch=32,
arc_epochs=5,
initial_decays=5,
convergence_decays=2):
assert initial_decays > convergence_decays, "%d <= %d" % (
initial_decays, convergence_decays)
train_stream, validation_stream, test_stream = data_streams
best_score_train = self.test(train_stream)
best_score_validation = self.test(validation_stream)
score_test = self.test(test_stream)
logging.debug(
"Baseline train/validation/test scores (random initialized weights): %.4f / %.4f / %.4f"
% (best_score_train, best_score_validation, score_test))
training_parameters = mlbase.TrainingParameters() \
.batch(batch) \
.epochs(arc_epochs) \
.convergence(False) \
.debug(True)
previous_loss = None
arc = -1
version = 0
self.save_parameters(model_dir, version, True)
initialized = False
converged = False
decays = 0
while not converged:
arc += 1
logging.debug("Train arc %d: %s" % (arc, training_parameters))
loss, score_train, score_validation = self._train_loop(
train_stream, validation_stream, training_parameters)
loss_change = self._change(previous_loss, loss,
lambda prev, curr: prev > curr)
train_change = self._change(best_score_train, score_train,
lambda prev, curr: prev < curr)
validation_change = self._change(best_score_validation,
score_validation,
lambda prev, curr: prev < curr)
logging.debug(
"Train arc %d: (loss, tr, va) (%s %.4f, %s %.4f, %s %.4f)" %
(arc, loss_change, loss, train_change, score_train,
validation_change, score_validation))
both_improved = score_train > best_score_train and score_validation > best_score_validation
if score_train > best_score_train or score_validation > best_score_validation:
previous_loss = loss
version += 1
initialized = True
self.save_parameters(model_dir, version, True)
# At least one improved.
if score_train > best_score_train:
best_score_train = score_train
if score_validation > best_score_validation:
best_score_validation = score_validation
else:
# The validation score didn't improve. Lets see where the test score is at.
score_test = self.test(test_stream)
logging.debug("Test score: %.4f" % score_test)
else:
# Neither improved.
# Load the best known version to continue training off of.
self.load_parameters(model_dir)
if not both_improved:
if decays >= convergence_decays if initialized else decays >= initial_decays:
converged = True
logging.debug("Converged" + (
"" if initialized else " without initialization!"))
else:
decays += 1
logging.debug("Decaying.. %d", decays)
training_parameters = training_parameters.decay(
initial=initialized)
else:
logging.debug("Reset decay")
decays = 0
# Load which ever version was marked as the latest as the final trained self.
self.load_parameters(model_dir)
logging.debug("Calculating final scores.")
score_train = self.test(train_stream, False)
score_validation = self.test(validation_stream, False)
score_test = self.test(test_stream, True)
logging.debug("(tr, va, te): (%.4f, %.4f, %.4f)" %
(score_train, score_validation, score_test))