def train_model(config): if config.start_date is not None: print("Training start date: ", config.start_date) if config.start_date is not None: print("Training end date: ", config.end_date) print("Loading training data from %s ..." % config.datafile) train_data = None valid_data = None if (config.validation_size > 0.0) or (config.split_date is not None): train_data, valid_data = data_utils.load_train_valid_data(config) else: train_data = data_utils.load_all_data(config, is_training_only=True) valid_data = train_data tf_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) with tf.Graph().as_default(), tf.Session(config=tf_config) as session: if config.seed is not None: tf.set_random_seed(config.seed) print("Constructing model ...") model = model_utils.get_model(session, config, verbose=True) params = model_utils.get_scaling_params(config, train_data, verbose=True) model.set_scaling_params(session, **params) noise_model = None if config.training_noise is not None: print("Training noise level: %.2f * 1-stdev" % config.training_noise) noise_model = NoiseModel(seed=config.seed, scaling_params=params, degree=config.training_noise) if config.early_stop is not None: print("Training will early stop without " "improvement after %d epochs." % config.early_stop) sys.stdout.flush() train_history = list() valid_history = list() lr = model.set_learning_rate(session, config.learning_rate) train_data.cache(verbose=True) valid_data.cache(verbose=True) for i in range(config.max_epoch): (train_mse, valid_mse) = run_epoch(session, model, train_data, valid_data, keep_prob=config.keep_prob, passes=config.passes, noise_model=noise_model, verbose=True) print(( 'Epoch: %d Train MSE: %.6f Valid MSE: %.6f Learning rate: %.4f' ) % (i + 1, train_mse, valid_mse, lr)) sys.stdout.flush() train_history.append(train_mse) valid_history.append(valid_mse) if re.match("Gradient|Momentum", config.optimizer): lr = model_utils.adjust_learning_rate(session, model, lr, config.lr_decay, train_history) if not os.path.exists(config.model_dir): print("Creating directory %s" % config.model_dir) os.mkdir(config.model_dir) if math.isnan(valid_mse): print("Training failed due to nan.") quit() elif stop_training(config, valid_history): print("Training stopped.") quit() else: if ((config.early_stop is None) or (valid_history[-1] <= min(valid_history))): model_utils.save_model(session, config, i)
def train_model(config): print("\nLoading training data ...") train_data, valid_data = data_utils.load_train_valid_data(config) if config.start_date is not None: print("Training start date: ", config.start_date) if config.start_date is not None: print("Training end date: ", config.end_date) tf_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) with tf.Graph().as_default(), tf.Session(config=tf_config) as session: if config.seed is not None: tf.set_random_seed(config.seed) print("\nConstructing model ...") model = model_utils.get_model(session, config, verbose=True) if config.data_scaler is not None: start_time = time.time() print("Calculating scaling parameters ...", end=' ') sys.stdout.flush() scaling_params = train_data.get_scaling_params(config.data_scaler) model.set_scaling_params(session, **scaling_params) print("done in %.2f seconds." % (time.time() - start_time)) #print(scaling_params['center']) #print(scaling_params['scale']) #exit(0) if config.early_stop is not None: print("Training will early stop without " "improvement after %d epochs." % config.early_stop) train_history = list() valid_history = list() lr = model.set_learning_rate(session, config.learning_rate) train_data.cache(verbose=True) valid_data.cache(verbose=True) for i in range(config.max_epoch): (train_mse, valid_mse) = run_epoch(session, model, train_data, valid_data, keep_prob=config.keep_prob, passes=config.passes, verbose=True) print(( 'Epoch: %d Train MSE: %.6f Valid MSE: %.6f Learning rate: %.4f' ) % (i + 1, train_mse, valid_mse, lr)) sys.stdout.flush() train_history.append(train_mse) valid_history.append(valid_mse) if re.match("Gradient|Momentum", config.optimizer): lr = model_utils.adjust_learning_rate(session, model, lr, config.lr_decay, train_history) if not os.path.exists(config.model_dir): print("Creating directory %s" % config.model_dir) os.mkdir(config.model_dir) chkpt_file_prefix = "training.ckpt" if model_utils.stop_training(config, valid_history, chkpt_file_prefix): print("Training stopped.") quit() else: checkpoint_path = os.path.join(config.model_dir, chkpt_file_prefix) if (valid_history[-1] == min(valid_history)): tf.train.Saver().save(session, checkpoint_path, global_step=i)
def train_model(config): if config.start_date is not None: print("Training start date: ", config.start_date) if config.start_date is not None: print("Training end date: ", config.end_date) print("Loading training data from %s ..."%config.datafile) train_data = None valid_data = None if (config.validation_size > 0.0) or (config.split_date is not None): train_data, valid_data = data_utils.load_train_valid_data(config) else: train_data = data_utils.load_all_data(config, is_training_only=True) valid_data = train_data tf_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) tf_config.gpu_options.allow_growth = True with tf.Graph().as_default(), tf.Session(config=tf_config) as session: if config.seed is not None: tf.set_random_seed(config.seed) print("Constructing model ...") model = model_utils.get_model(session, config, verbose=True) if config.data_scaler is not None: start_time = time.time() print("Calculating scaling parameters ...", end=' '); sys.stdout.flush() scaling_params = train_data.get_scaling_params(config.data_scaler) model.set_scaling_params(session,**scaling_params) print("done in %.2f seconds."%(time.time() - start_time)) print("%-10s %-6s %-6s"%('feature','mean','std')) for i in range(len(train_data.feature_names)): center = "%.4f"%scaling_params['center'][i]; scale = "%.4f"%scaling_params['scale'][i]; print("%-10s %-6s %-6s"%(train_data.feature_names[i], center,scale)) sys.stdout.flush() if config.early_stop is not None: print("Training will early stop without " "improvement after %d epochs."%config.early_stop) train_history = list() valid_history = list() lr = model.set_learning_rate(session, config.learning_rate) train_data.cache(verbose=True) valid_data.cache(verbose=True) for i in range(config.max_epoch): # MVE Epoch if config.UQ_model_type == 'MVE': (train_mse, train_mse_var, valid_mse, valid_mse_var) = run_epoch_mve(session, model, train_data, valid_data, keep_prob=config.keep_prob, passes=config.passes, verbose=True) # Status to check if valid mse is nan, used to stop training if math.isnan(valid_mse): is_metric_nan = True else: is_metric_nan = False print('Epoch: %d Train MSE: %.8f Valid MSE: %.8f Learning rate: %.4f' % (i + 1, train_mse, valid_mse, lr)) print('Epoch: %d Train MSE_w_variance: %.8f Valid MSE_w_variance: %.8f Learning rate: %.4f' % (i + 1, train_mse_var, valid_mse_var, lr)) sys.stdout.flush() train_history.append(train_mse_var) valid_history.append(valid_mse_var) # PIE Epoch elif config.UQ_model_type == 'PIE': (train_mpiw, train_picp, train_picp_loss, valid_mpiw, valid_picp, valid_picp_loss) = \ run_epoch_pie(session, model, train_data, valid_data, keep_prob=config.keep_prob, passes=config.passes, verbose=True) train_loss = train_mpiw + config.picp_lambda*train_picp_loss valid_loss = valid_mpiw + config.picp_lambda*valid_picp_loss # Status to check if valid loss is nan, used to stop training if math.isnan(valid_loss): is_metric_nan = True else: is_metric_nan = False print('Epoch: %d Train MPIW: %.8f Valid MPIW: %.8f Learning rate: %.4f' % (i + 1, train_mpiw, valid_mpiw, lr)) print('Epoch: %d Train PICP: %.8f Valid PICP: %.8f' % (i + 1, train_picp, valid_picp)) print('Epoch: %d Train LOSS: %.8f Valid LOSS: %.8f' % (i + 1, train_loss, valid_loss )) sys.stdout.flush() train_history.append(train_loss) valid_history.append(valid_loss) if re.match("Gradient|Momentum", config.optimizer): lr = model_utils.adjust_learning_rate(session, model, lr, config.lr_decay, train_history) if not os.path.exists(config.model_dir): print("Creating directory %s" % config.model_dir) os.mkdir(config.model_dir) if is_metric_nan: print("Training failed due to nan.") quit() elif stop_training(config, valid_history): print("Training stopped.") quit() else: if ( (config.early_stop is None) or (valid_history[-1] <= min(valid_history)) ): model_utils.save_model(session, config, i)
def train_model(config, verbose=True): print("\nLoading training data ...") train_data, valid_data = load_train_valid_data(config, verbose) if config.start_date is not None: print("Training start date: ", config.start_date) if config.start_date is not None: print("Training end date: ", config.end_date) tf_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, device_count={'GPU': 0}) with tf.Graph().as_default(), tf.Session(config=tf_config) as session: if config.seed is not None: tf.set_random_seed(config.seed) print("\nConstructing model ...") model = model_utils.get_model(session, config, verbose=verbose) if config.data_scaler is not None: start_time = time.time() print("Calculating scaling parameters ...", end=' ') sys.stdout.flush() scaling_params = train_data.get_scaling_params(config.data_scaler) model.set_scaling_params(session, **scaling_params) print("done in %.2f seconds." % (time.time() - start_time)) if config.early_stop is not None: print("Training will early stop without " "improvement after %d epochs." % config.early_stop) train_history = list() valid_history = list() lr = model.set_learning_rate(session, config.learning_rate) train_data.cache(verbose=verbose) valid_data.cache(verbose=verbose) for i in range(config.max_epoch): (train_mse, valid_mse) = run_epoch(session, model, train_data, valid_data, keep_prob=config.keep_prob, passes=config.passes, verbose=verbose) if verbose: print(('Epoch: %d Train MSE: %.6f Valid MSE: %.6f Learning' 'rate: %.4f') % (i + 1, train_mse, valid_mse, lr)) sys.stdout.flush() train_history.append(train_mse) valid_history.append(valid_mse) if re.match("Gradient|Momentum", config.optimizer): lr = model_utils.adjust_learning_rate(session, model, lr, config.lr_decay, train_history) return train_history, valid_history