with open(str(save_dir / ("config_%d.json" % config.initial_epoch)), "w") as fp: json.dump(vars(config), fp, sort_keys=True, indent=4) # Set up training callbacks checkpoint = ModelCheckpoint(filepath=str(file_path), monitor="val_acc", verbose=1) tensor_board = TensorBoard(log_dir=save_dir, histogram_freq=0, batch_size=config.batch_size, write_graph=True, write_grads=False, write_images=False, update_freq=2500) tensor_board.samples_seen = config.initial_epoch * len(x_train) tensor_board.samples_seen_at_last_write = config.initial_epoch * len(x_train) lr_reducer = ReduceLROnPlateau(monitor="val_loss", factor=config.lr_reduce_factor, cooldown=0, patience=config.lr_reduce_patience, min_lr=config.lr_reduce_min, verbose=1) early_stopping = EarlyStopping(monitor="val_loss", patience=config.early_stop_patience, verbose=1) callbacks = [checkpoint, tensor_board] if (config.lr_reduce): callbacks.append(lr_reducer) if (config.early_stop):
batch_size = 256 epochs = epoch + args.epochs train = PositionSequence(training_database, batch_size=batch_size, rotate=epoch * config["batches_per_epoch"]) val = PositionSequence(validation_database, batch_size=batch_size) extra: Dict[str, PositionSequence] = { n: PositionSequence(db, batch_size=batch_size) for n, db in extra_databases.items() } tb = TensorBoard(tensorboard_dir, update_freq=batch_size * config["batches_per_epoch"] / 4) tb.samples_seen = epoch * config["batches_per_epoch"] * batch_size tb.samples_seen_at_last_write = tb.samples_seen callbacks = [ LambdaCallback(on_epoch_end=save_model), LambdaCallback(on_epoch_end=validate_extra_datasets), tb ] network.training_model.fit_generator( train, epochs=epochs, callbacks=callbacks, verbose=2 if not is_atty else 1, validation_data=val, shuffle=False, workers=1, use_multiprocessing=True,