def load_train_valid_data(config, verbose): """ Returns train_data and valid_data, both as BatchGenerator objects. """ data_path = data_utils.get_data_path(config.data_dir, config.datafile) batches = BatchGenerator(data_path, config, verbose=verbose) train_data = batches.train_batches() valid_data = batches.valid_batches() return train_data, valid_data
def train_model(config): if config.start_date is not None: print("Training start date: ", config.start_date) if config.start_date is not None: print("Training end date: ", config.end_date) print("Loading training data from %s ..." % config.datafile) train_data = None valid_data = None data_path = os.path.join(config.data_dir, config.datafile) batches = BatchGenerator(data_path, config, is_training_only=True) train_data = batches.train_batches(verbose=True) valid_data = batches.valid_batches(verbose=True) tf_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) with tf.Graph().as_default(), tf.Session(config=tf_config) as session: if config.seed is not None: tf.set_random_seed(config.seed) print("Constructing model ...") model = model_utils.get_model(session, config, verbose=True) params = model_utils.get_scaling_params(config, train_data, verbose=True) model.set_scaling_params(session, **params) noise_model = None if config.early_stop is not None: print("Training will early stop without " "improvement after %d epochs." % config.early_stop) sys.stdout.flush() train_history = list() valid_history = list() lr = model.set_learning_rate(session, config.learning_rate) train_data.cache(verbose=True) valid_data.cache(verbose=True) for i in range(config.max_epoch): (train_mse, valid_mse) = run_epoch(session, model, train_data, valid_data, keep_prob=config.keep_prob, passes=config.passes, noise_model=noise_model, verbose=True) print(( 'Epoch: %d Train MSE: %.6f Valid MSE: %.6f Learning rate: %.4f' ) % (i + 1, train_mse, valid_mse, lr)) sys.stdout.flush() train_history.append(train_mse) valid_history.append(valid_mse) if re.match("Gradient|Momentum", config.optimizer): lr = model_utils.adjust_learning_rate(session, model, lr, config.lr_decay, train_history) if not os.path.exists(config.model_dir): print("Creating directory %s" % config.model_dir) os.mkdir(config.model_dir) if math.isnan(valid_mse): print("Training failed due to nan.") quit() elif stop_training(config, valid_history): print("Training stopped.") quit() else: if ((config.early_stop is None) or (valid_history[-1] <= min(valid_history))): model_utils.save_model(session, config, i)
# If cached data doesn't exist, build it ############################################################################ if not os.path.exists(cache_path) or config.use_cache is False: print("Generating Data from Scratch") config.end_date = 999901 data_bg = BatchGenerator(train_path, config, config.batch_size, config.num_unrollings, validation_size=config.validation_size, randomly_sample=False) train_bg = data_bg.train_batches() valid_bg = data_bg.valid_batches() print("Grabbing tabular data from batch generator") X_train_full, Y_train_full, dates_train = get_tabular_data(train_bg) X_valid_full, Y_valid_full, dates_valid = get_tabular_data(valid_bg) print("Saving tabular data to cache") # JDA 10/27/16: Save these objects to cache here if not os.path.exists(cache_path): os.mkdir(cache_path) np.save(os.path.join(cache_path, 'X_train_full.npy'), X_train_full) np.save(os.path.join(cache_path, 'Y_train_full.npy'), Y_train_full) np.save(os.path.join(cache_path, 'X_valid_full.npy'), X_valid_full) np.save(os.path.join(cache_path, 'Y_valid_full.npy'), Y_valid_full) np.save(os.path.join(cache_path, 'dates_train.npy'), dates_train) np.save(os.path.join(cache_path, 'dates_valid.npy'), dates_valid)
############################################################################ # If cached data doesn't exist, build it ############################################################################ if not os.path.exists(cache_path) or config.use_cache is False: print("Generating Data from Scratch") config.end_date = 999901 data_bg = BatchGenerator(train_path, config, config.batch_size, config.num_unrollings, validation_size=config.validation_size, randomly_sample=False) train_bg = data_bg.train_batches() valid_bg = data_bg.valid_batches() print("Grabbing tabular data from batch generator") X_train_full, Y_train_full, dates_train = get_tabular_data(train_bg) X_valid_full, Y_valid_full, dates_valid = get_tabular_data(valid_bg) print("Saving tabular data to cache") # JDA 10/27/16: Save these objects to cache here if not os.path.exists(cache_path): os.mkdir(cache_path) np.save(os.path.join(cache_path, 'X_train_full.npy'), X_train_full ) np.save(os.path.join(cache_path, 'Y_train_full.npy'), Y_train_full ) np.save(os.path.join(cache_path, 'X_valid_full.npy'), X_valid_full ) np.save(os.path.join(cache_path, 'Y_valid_full.npy'), Y_valid_full ) np.save(os.path.join(cache_path, 'dates_train.npy'), dates_train ) np.save(os.path.join(cache_path, 'dates_valid.npy'), dates_valid )