def read_data(dataset_path, normalization_method, past_history_factor): # read normalization params norm_params = None with open( os.path.normpath(dataset_path) + "/{}/norm_params.json".format(normalization_method), "r", ) as read_file: norm_params = json.load(read_file) # read training / validation data tmp_data_path = os.path.normpath(dataset_path) + "/{}/{}/".format( normalization_method, past_history_factor) x_train = np.load(tmp_data_path + "x_train.np.npy") y_train = np.load(tmp_data_path + "y_train.np.npy") x_test = np.load(tmp_data_path + "x_test.np.npy") y_test = np.load(tmp_data_path + "y_test.np.npy") y_test_denorm = np.asarray([ denormalize(y_test[i], norm_params[i], normalization_method) for i in range(y_test.shape[0]) ]) print("TRAINING DATA") print("Input shape", x_train.shape) print("Output_shape", y_train.shape) print("TEST DATA") print("Input shape", x_test.shape) print("Output_shape", y_test.shape) return x_train, y_train, x_test, y_test, y_test_denorm, norm_params
def train_trees(model_name, iter_params, x_train, y_train, x_test, norm_params, normalization_method): model = create_model_ml(model_name, iter_params) x_train2 = x_train.reshape(x_train.shape[0], x_train.shape[1] * x_train.shape[2]) print('x_train: {} -> {}'.format(x_train.shape, x_train2.shape)) training_time_0 = time.time() model.fit(x_train2, y_train) training_time = time.time() - training_time_0 x_test2 = x_test.reshape(x_test.shape[0], x_test.shape[1] * x_test.shape[2]) print('x_test: {} -> {}'.format(x_test.shape, x_test2.shape)) test_time_0 = time.time() test_forecast = model.predict(x_test2) test_time = time.time() - test_time_0 for i in range(test_forecast.shape[0]): nparams = norm_params[0] test_forecast[i] = denormalize( test_forecast[i], nparams, method=normalization_method, ) return test_forecast, training_time, test_time
def _run_experiment( gpu_device, dataset, dataset_path, results_path, csv_filepath, metrics, epochs, normalization_method, past_history_factor, max_steps_per_epoch, batch_size, learning_rate, model_name, model_index, model_args, ): import gc import tensorflow as tf from models import create_model tf.keras.backend.clear_session() def select_gpu_device(gpu_number): gpus = tf.config.experimental.list_physical_devices("GPU") if len(gpus) >= 2 and gpu_number is not None: device = gpus[gpu_number] tf.config.experimental.set_memory_growth(device, True) tf.config.experimental.set_visible_devices(device, "GPU") select_gpu_device(gpu_device) results = read_results_file(csv_filepath, metrics) x_train, y_train, x_test, y_test, y_test_denorm, norm_params = read_data( dataset_path, normalization_method, past_history_factor ) x_train = tf.convert_to_tensor(x_train) y_train = tf.convert_to_tensor(y_train) x_test = tf.convert_to_tensor(x_test) y_test = tf.convert_to_tensor(y_test) y_test_denorm = tf.convert_to_tensor(y_test_denorm) forecast_horizon = y_test.shape[1] past_history = x_test.shape[1] steps_per_epoch = min( int(np.ceil(x_train.shape[0] / batch_size)), max_steps_per_epoch, ) optimizer = tf.optimizers.Adam(learning_rate=learning_rate) model = create_model( model_name, x_train.shape, output_size=forecast_horizon, optimizer=optimizer, loss="mae", **model_args ) print(model.summary()) training_time_0 = time.time() history = model.fit( x_train, y_train, batch_size=batch_size, epochs=epochs, steps_per_epoch=steps_per_epoch, validation_data=(x_test, y_test), shuffle=True, ) training_time = time.time() - training_time_0 # Get validation metrics test_time_0 = time.time() test_forecast = model(x_test).numpy() test_time = time.time() - test_time_0 for i in range(test_forecast.shape[0]): nparams = norm_params[0] test_forecast[i] = denormalize( test_forecast[i], nparams, method=normalization_method, ) if metrics: test_metrics = evaluate(y_test_denorm, test_forecast, metrics) else: test_metrics = {} # Save results predictions_path = "{}/{}/{}/{}/{}/{}/{}/{}/".format( results_path, dataset, normalization_method, past_history_factor, epochs, batch_size, learning_rate, model_name, ) if not os.path.exists(predictions_path): os.makedirs(predictions_path) np.save( predictions_path + str(model_index) + ".npy", test_forecast, ) results = results.append( { "DATASET": dataset, "MODEL": model_name, "MODEL_INDEX": model_index, "MODEL_DESCRIPTION": str(model_args), "FORECAST_HORIZON": forecast_horizon, "PAST_HISTORY_FACTOR": past_history_factor, "PAST_HISTORY": past_history, "BATCH_SIZE": batch_size, "EPOCHS": epochs, "STEPS": steps_per_epoch, "OPTIMIZER": "Adam", "LEARNING_RATE": learning_rate, "NORMALIZATION": normalization_method, "TEST_TIME": test_time, "TRAINING_TIME": training_time, **test_metrics, "LOSS": str(history.history["loss"]), "VAL_LOSS": str(history.history["val_loss"]), }, ignore_index=True, ) results.to_csv( csv_filepath, sep=";", ) gc.collect() del model, x_train, x_test, y_train, y_test, y_test_denorm, test_forecast
def generate_dataset(args): dataset, norm_method, past_history_factor = args train_url = DATASETS[dataset]["train"] test_url = DATASETS[dataset]["test"] if not os.path.exists( "../data/{}/train.csv".format(dataset)) or not os.path.exists( "../data/{}/test.csv".format(dataset)): if not os.path.exists("../data/{}".format(dataset)): os.system("mkdir -p ../data/{}".format(dataset)) os.system("wget -O ../data/{}/train.csv {}".format(dataset, train_url)) os.system("wget -O ../data/{}/test.csv {}".format(dataset, test_url)) if not os.path.exists("../data/{}/{}/{}/".format(dataset, norm_method, past_history_factor)): os.system("mkdir -p ../data/{}/{}/{}/".format(dataset, norm_method, past_history_factor)) # Read data train = read_ts_dataset("../data/{}/train.csv".format(dataset)) test = read_ts_dataset("../data/{}/test.csv".format(dataset)) print("Shape test", test.shape) forecast_horizon = test.shape[1] print( dataset, { "Number of time series": train.shape[0], "Max length": np.max([ts.shape[0] for ts in train]), "Min length": np.min([ts.shape[0] for ts in train]), "Forecast Horizon": forecast_horizon, }, ) #Format training and test input/output data using the moving window strategy past_history = int(forecast_horizon * past_history_factor) # Normalize data train, test, norm_params = normalize_dataset(train, test, norm_method, dtype="float32") norm_params_json = [{k: float(p[k]) for k in p} for p in norm_params] norm_params_json = json.dumps(norm_params_json) with open("../data/{}/{}/norm_params.json".format(dataset, norm_method), "w") as f: f.write(norm_params_json) invalidParams = [] for i in range(len(train)): if len(train[i]) < past_history: invalidParams.append(i) x_train, y_train, x_test, y_test = moving_windows_preprocessing( train, test, past_history, forecast_horizon, np.float32, n_cores=NUM_CORES) y_test_denorm = np.copy(y_test) j = 0 for i, nparams in enumerate(norm_params): if i not in invalidParams: y_test_denorm[j] = denormalize(y_test[j], nparams, method=norm_method) j += 1 print("TRAINING DATA") print("Input shape", x_train.shape) print("Output_shape", y_train.shape) print() print("TEST DATA") print("Input shape", x_test.shape) print("Output_shape", y_test.shape) np.save( "../data/{}/{}/{}/x_train.np".format(dataset, norm_method, past_history_factor), x_train, ) np.save( "../data/{}/{}/{}/y_train.np".format(dataset, norm_method, past_history_factor), y_train, ) np.save( "../data/{}/{}/{}/x_test.np".format(dataset, norm_method, past_history_factor), x_test, ) np.save( "../data/{}/{}/{}/y_test.np".format(dataset, norm_method, past_history_factor), y_test, ) np.save( "../data/{}/{}/{}/y_test_denorm.np".format(dataset, norm_method, past_history_factor), y_test_denorm, ) # Save indexes of invalid normalization parametes if invalidParams != []: invalidParams = np.asarray(invalidParams) np.save( "../data/{}/{}/{}/invalidParams.np".format(dataset, norm_method, past_history_factor), invalidParams, )
def generate_dataset(args): dataset, norm_method, past_history_factor = args train_url = DATASETS[dataset]["train"] test_url = DATASETS[dataset]["test"] train = read_ts_dataset("../data/{}/train.csv".format(dataset)) test = read_ts_dataset("../data/{}/test.csv".format(dataset)) forecast_horizon = 24 #test.shape[1] print( dataset, { "Number of time series": train.shape[0], "Max length": np.max([ts.shape[0] for ts in train]), "Min length": np.min([ts.shape[0] for ts in train]), "Forecast Horizon": forecast_horizon, }, ) # Normalize data train, test, norm_params = normalize_dataset(train, test, norm_method, dtype="float32") norm_params_json = [{k: float(p[k]) for k in p} for p in norm_params] norm_params_json = json.dumps(norm_params_json) with open("../data/{}/{}/norm_params.json".format(dataset, norm_method), "w") as f: f.write(norm_params_json) # Format training and test input/output data using the moving window strategy past_history = int(forecast_horizon * past_history_factor) x_train, y_train, x_test, y_test = moving_windows_preprocessing( train, test, past_history, forecast_horizon, np.float32, n_cores=NUM_CORES) y_test_denorm = np.copy(y_test) #i = 0 for i in range(y_test.shape[0]): y_test_denorm[i] = denormalize(y_test[i], norm_params[0], method=norm_method) print("TRAINING DATA") print("Input shape", x_train.shape) print("Output_shape", y_train.shape) print() print("TEST DATA") print("Input shape", x_test.shape) print("Output_shape", y_test.shape) np.save( "../data/{}/{}/{}/x_train.np".format(dataset, norm_method, past_history_factor), x_train, ) np.save( "../data/{}/{}/{}/y_train.np".format(dataset, norm_method, past_history_factor), y_train, ) np.save( "../data/{}/{}/{}/x_test.np".format(dataset, norm_method, past_history_factor), x_test, ) np.save( "../data/{}/{}/{}/y_test.np".format(dataset, norm_method, past_history_factor), y_test, ) np.save( "../data/{}/{}/{}/y_test_denorm.np".format(dataset, norm_method, past_history_factor), y_test_denorm, )
def _run_experiment_transformer( gpu_device, dataset, dataset_path, results_path, csv_filepath, metrics, epochs, normalization_method, past_history_factor, max_steps_per_epoch, batch_size, learning_rate, model_name, model_index, model_args, ): print("Start _run_experiment_transformer") import gc from models import create_model import torch from pytorch_lightning import Trainer, seed_everything from torch.utils.data import DataLoader, TensorDataset results = read_results_file(csv_filepath, metrics) x_train, y_train, x_test, y_test, y_test_denorm, norm_params = read_data( dataset_path, normalization_method, past_history_factor) forecast_horizon = y_test.shape[1] past_history = x_test.shape[1] steps_per_epoch = min( int(np.ceil(x_train.shape[0] / batch_size)), max_steps_per_epoch, ) x_train = torch.from_numpy(x_train).float() y_train = torch.from_numpy(y_train).float().unsqueeze(-1) x_test = torch.from_numpy(x_test).float() y_test = torch.from_numpy(y_test).float().unsqueeze(-1) train_dataset = TensorDataset(x_train, y_train) val_dataset = TensorDataset(x_test, y_test) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False) test_loader = DataLoader(val_dataset, batch_size=256, shuffle=False) #seed_everything(42,workers=true) trainer = Trainer(max_epochs=epochs, gpus=[gpu_device], checkpoint_callback=False) model = create_model(model_name, x_train.shape, output_size=forecast_horizon, **model_args) training_time_0 = time.time() trainer.fit(model, train_loader, val_loader) training_time = time.time() - training_time_0 print("End training") train_loss = float(trainer.callback_metrics["train_loss"].to("cpu")) val_loss = float(trainer.callback_metrics["val_loss"].to("cpu")) print("loss saved") def predictMultiStepRegresive(model, x_test, nSteps): ''' Inference for autoregresive models ''' with torch.no_grad(): encoderInput = x_test decoderInput = x_test[:, -1, :] decoderInput = decoderInput.unsqueeze(-1) for _ in range( nSteps ): # We append the prediction on step 0 to the original input to get the input for step 1 and so on. out = model.forward( encoderInput, decoderInput ) # We don't need mask for evaluation, we are not giving the model any future input. lastPrediction = out[:, -1].detach( ) #detach() is quite important, otherwise we will keep the variable "out" in memory and cause an out of memory error. lastPrediction = lastPrediction.unsqueeze(-1) decoderInput = torch.cat((decoderInput, lastPrediction), 1) decoderInput = decoderInput.squeeze(-1) return decoderInput[:, 1:] # Only use "device" for inference, as Pytorch Lighting already handles it for training device = torch.device( "cuda:" + str(gpu_device) if torch.cuda.is_available() else "cpu") model.to(device) model.eval() x_test = x_test.to(device) if model_name.endswith("AR"): ''' Autoregresive inference ''' test_time_0 = time.time() test_forecast = predictMultiStepRegresive(model, x_test, y_test.shape[1]) test_time = time.time() - test_time_0 else: ''' Non-Autoregresive inference ''' test_time_0 = time.time() test_forecast = model(x_test) test_time = time.time() - test_time_0 print("End test") test_forecast = test_forecast.detach().to("cpu").numpy() for i, nparams in enumerate(norm_params): test_forecast[i] = denormalize( test_forecast[i], nparams, method=normalization_method, ) if metrics: test_metrics = evaluate(y_test_denorm, test_forecast, metrics) print(test_metrics) else: print("Metrics empty") test_metrics = {} # Save results predictions_path = "{}/{}/{}/{}/{}/{}/{}/{}/".format( results_path, dataset, normalization_method, past_history_factor, epochs, batch_size, learning_rate, model_name, ) if not os.path.exists(predictions_path): os.makedirs(predictions_path) np.save( predictions_path + str(model_index) + ".npy", test_forecast, ) results = results.append( { "DATASET": dataset, "MODEL": model_name, "MODEL_INDEX": model_index, "MODEL_DESCRIPTION": str(model_args), "FORECAST_HORIZON": forecast_horizon, "PAST_HISTORY_FACTOR": past_history_factor, "PAST_HISTORY": past_history, "BATCH_SIZE": batch_size, "EPOCHS": epochs, "STEPS": steps_per_epoch, "OPTIMIZER": "CustomAdam", "LEARNING_RATE": learning_rate, "NORMALIZATION": normalization_method, "TEST_TIME": test_time, "TRAINING_TIME": training_time, **test_metrics, "LOSS": str(train_loss), "VAL_LOSS": str(val_loss), }, ignore_index=True, ) results.to_csv( csv_filepath, sep=";", ) gc.collect() del model, x_train, x_test, y_train, y_test, y_test_denorm, test_forecast
def _run_experiment_transformer( gpu_device, dataset, dataset_path, results_path, csv_filepath, metrics, epochs, normalization_method, past_history_factor, max_steps_per_epoch, batch_size, learning_rate, model_name, model_index, model_args, ): print("Start _run_experiment_transformer") import gc from models import create_model from transformerOptimizer import get_std_opt from transformerTraining import train from transformerTraining import predictMultiStep from transformerTraining import predictMultiStepBatching import torch from torch.nn import L1Loss results = read_results_file(csv_filepath, metrics) x_train, y_train, x_test, y_test, y_test_denorm, norm_params = read_data( dataset_path, normalization_method, past_history_factor) forecast_horizon = y_test.shape[1] past_history = x_test.shape[1] device = torch.device( "cuda:" + str(gpu_device) if torch.cuda.is_available() else "cpu") steps_per_epoch = min( int(np.ceil(x_train.shape[0] / batch_size)), max_steps_per_epoch, ) x_train = torch.from_numpy(x_train).float().to(device) y_train = torch.from_numpy(y_train).float().to(device) x_test = torch.from_numpy(x_test).float().to(device) y_test = torch.from_numpy(y_test).float().to(device) y_test_denorm = torch.from_numpy(y_test_denorm).float() model = create_model(model_name, x_train.shape, **model_args) if learning_rate == "Noam": model_opt = get_std_opt(model) optimizerName = "Noam" else: model_opt = torch.optim.Adam(model.parameters(), lr=learning_rate) optimizerName = "Adam" model.to(device) criterion = L1Loss() # mean absolute error training_time_0 = time.time() trainLoss, valLoss = train(model, x_train, y_train, x_test, y_test, epochs, steps_per_epoch, criterion, model_opt, batch_size) training_time = time.time() - training_time_0 # Get validation metrics test_time_0 = time.time() if y_test.shape[0] > 256: #Batch inference to avoid out of memory error test_forecast = predictMultiStepBatching(x_test, model, y_test.shape[1]) else: test_forecast = predictMultiStep(x_test, model, y_test.shape[1]) test_time = time.time() - test_time_0 test_time = test_time / x_train.shape[0] normalized_test_forecast = test_forecast.numpy().copy() for i, nparams in enumerate(norm_params): test_forecast[i] = denormalize( test_forecast[i], nparams, method=normalization_method, ) if metrics: print("y_test: ", y_test_denorm.numpy()) print("test_forecast: ", test_forecast) test_metrics = evaluate(y_test_denorm.numpy(), test_forecast.numpy(), metrics) print(test_metrics) else: print("Metrics empty") test_metrics = {} # Save results predictions_path = "{}/{}/{}/{}/{}/{}/{}/{}/".format( results_path, dataset, normalization_method, past_history_factor, epochs, batch_size, learning_rate, model_name, ) if not os.path.exists(predictions_path): os.makedirs(predictions_path) np.save( predictions_path + str(model_index) + ".npy", test_forecast, ) np.save( predictions_path + "Normalize" + str(model_index) + ".npy", normalized_test_forecast, ) results = results.append( { "DATASET": dataset, "MODEL": model_name, "MODEL_INDEX": model_index, "MODEL_DESCRIPTION": str(model_args), "FORECAST_HORIZON": forecast_horizon, "PAST_HISTORY_FACTOR": past_history_factor, "PAST_HISTORY": past_history, "BATCH_SIZE": batch_size, "EPOCHS": epochs, "STEPS": steps_per_epoch, "OPTIMIZER": optimizerName, "LEARNING_RATE": learning_rate, "NORMALIZATION": normalization_method, "TEST_TIME": test_time, "TRAINING_TIME": training_time, **test_metrics, "LOSS": str(trainLoss), "VAL_LOSS": str(valLoss), }, ignore_index=True, ) results.to_csv( csv_filepath, sep=";", ) gc.collect() del model, x_train, x_test, y_train, y_test, y_test_denorm, test_forecast