def train(self, data, epochs, batch=8):
xTrain, yTrain = data.train()
xValid, yValid = data.valid()
while epochs > 0:
console.log("Training for", epochs, "epochs on", len(xTrain),
"examples")
self.model.fit(xTrain,
yTrain,
batch_size=batch,
epochs=epochs,
validation_data=(xValid, yValid))
console.notify(
str(epochs) + " Epochs Complete!", "Training on", data.inPath,
"with size", batch)
while True:
try:
epochs = int(
input("How many more epochs should we train for? "))
break
except ValueError:
console.warn(
"Oops, number parse failed. Try again, I guess?")
if epochs > 0:
save = input("Should we save intermediate weights [y/n]? ")
if not save.lower().startswith("n"):
weightPath = ''.join(
random.choice(string.digits)
for _ in range(16)) + ".h5"
console.log("Saving intermediate weights to", weightPath)
self.saveWeights(weightPath)
def train(self, data, epochs, batch=8, start_epoch=0):
x_train, y_train = data.train()
x_valid, y_valid = data.valid()
self.x_valid, self.y_valid = x_valid, y_valid
checkpointer = Checkpointer(self)
checkpoints = checkpointer.get()
if self.config.batch_generator != "keras":
batch_generator = Batch().get()
if self.config.epoch_steps:
epoch_steps = self.config.epoch_steps
else:
epoch_steps = remove_track_boundaries(x_train).shape[0]
epoch_steps = epoch_steps // batch
while epochs > 0:
end_epoch = start_epoch + epochs
console.log("Training for", epochs, "epochs on",
epoch_steps * batch, "examples")
console.log("Validate on", len(x_valid), "examples")
if self.config.batch_generator == "keras":
x_train = remove_track_boundaries(x_train)
y_train = remove_track_boundaries(y_train)
history = self.model.fit(x_train,
y_train,
batch_size=batch,
initial_epoch=start_epoch,
epochs=end_epoch,
validation_data=(x_valid, y_valid),
callbacks=checkpoints)
else:
history = self.model.fit_generator(
batch_generator(x_train, y_train, batch_size=batch),
initial_epoch=start_epoch,
epochs=end_epoch,
steps_per_epoch=epoch_steps,
validation_data=(x_valid, y_valid),
callbacks=checkpoints)
console.notify(
str(epochs) + " Epochs Complete!", "Training on", data.in_path,
"with size", batch)
start_epoch += epochs
if self.config.quit:
break
else:
while True:
try:
epochs = int(
input("How many more epochs should we train for?"))
break
except ValueError:
console.warn(
"Oops, number parse failed. Try again, I guess?")
if epochs > 0:
save = input("Should we save intermediate weights [y/n]? ")
if not save.lower().startswith("n"):
weight_path = ''.join(
random.choice(string.digits)
for _ in range(16)) + ".h5"
os.path.join(os.path.dirname(config.weights),
weight_path)
console.log("Saving intermediate weights to",
weight_path)
self.save_weights(weight_path)
return history