def train(self, data, epochs, batch=8): xTrain, yTrain = data.train() xValid, yValid = data.valid() while epochs > 0: console.log("Training for", epochs, "epochs on", len(xTrain), "examples") self.model.fit(xTrain, yTrain, batch_size=batch, epochs=epochs, validation_data=(xValid, yValid)) console.notify( str(epochs) + " Epochs Complete!", "Training on", data.inPath, "with size", batch) while True: try: epochs = int( input("How many more epochs should we train for? ")) break except ValueError: console.warn( "Oops, number parse failed. Try again, I guess?") if epochs > 0: save = input("Should we save intermediate weights [y/n]? ") if not save.lower().startswith("n"): weightPath = ''.join( random.choice(string.digits) for _ in range(16)) + ".h5" console.log("Saving intermediate weights to", weightPath) self.saveWeights(weightPath)
def train(self, data, epochs, batch=8, start_epoch=0): x_train, y_train = data.train() x_valid, y_valid = data.valid() self.x_valid, self.y_valid = x_valid, y_valid checkpointer = Checkpointer(self) checkpoints = checkpointer.get() if self.config.batch_generator != "keras": batch_generator = Batch().get() if self.config.epoch_steps: epoch_steps = self.config.epoch_steps else: epoch_steps = remove_track_boundaries(x_train).shape[0] epoch_steps = epoch_steps // batch while epochs > 0: end_epoch = start_epoch + epochs console.log("Training for", epochs, "epochs on", epoch_steps * batch, "examples") console.log("Validate on", len(x_valid), "examples") if self.config.batch_generator == "keras": x_train = remove_track_boundaries(x_train) y_train = remove_track_boundaries(y_train) history = self.model.fit(x_train, y_train, batch_size=batch, initial_epoch=start_epoch, epochs=end_epoch, validation_data=(x_valid, y_valid), callbacks=checkpoints) else: history = self.model.fit_generator( batch_generator(x_train, y_train, batch_size=batch), initial_epoch=start_epoch, epochs=end_epoch, steps_per_epoch=epoch_steps, validation_data=(x_valid, y_valid), callbacks=checkpoints) console.notify( str(epochs) + " Epochs Complete!", "Training on", data.in_path, "with size", batch) start_epoch += epochs if self.config.quit: break else: while True: try: epochs = int( input("How many more epochs should we train for?")) break except ValueError: console.warn( "Oops, number parse failed. Try again, I guess?") if epochs > 0: save = input("Should we save intermediate weights [y/n]? ") if not save.lower().startswith("n"): weight_path = ''.join( random.choice(string.digits) for _ in range(16)) + ".h5" os.path.join(os.path.dirname(config.weights), weight_path) console.log("Saving intermediate weights to", weight_path) self.save_weights(weight_path) return history