def __init__(self, params, plot=True): self.config = Config() self.plotting = Plotting() self.params = params self.plot = plot # Number of Conditioning, Random and Prediction returns self.num_c = params["num_c"] self.num_z = params["num_z"] self.num_o = params["num_o"] self.num_tenors = params["num_tenors"] optimizer = Adam(1e-5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss=params["loss"], optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generates imgs condition = Input(shape=(self.num_c, self.num_tenors)) noise = Input(shape=(self.num_z, self.num_tenors)) img = self.generator([condition, noise]) # For the combined model we will only train the generator self.discriminator.trainable = False # The discriminator takes generated images as input and determines validity validity = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator self.combined = Model([condition, noise], validity) self.combined.compile(loss=params["loss"], optimizer=optimizer)
def simulate(self): plotting = Plotting() old_rates = self.model.rates plotting.plot_3d("AMModel_input_data", old_rates) plotting.plot_2d(old_rates[-1, :], "AMModel_input_data_first") tenors = self.model.tenors obs_time = self.model.obs_time print("tenors", tenors) print("obs_time", obs_time) print("old_rates", old_rates) self.model.make_data() rates = self.model.rates print("new rates", rates) plotting.plot_3d("AMModel_test", rates) # , maturities=tenors, time=obs_time print("made data")
def simulate(): plotting = Plotting() preprocess_normalisation = PreprocessData() preprocess_normalisation.enable_normalisation_scaler = True preprocess_normalisation.feature_range = [-1, 1] # preprocess_normalisation.enable_ignore_price = True # 1. get data and apply normalisation sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess_normalisation.get_data( ) all_training_scaled = np.vstack(sets_training_scaled) ae_params = { 'input_dim': sets_training_scaled[0].shape[1], # 56 'latent_dim': 3, 'hidden_layers': ( 56, 40, 28, 12, 4, ), 'leaky_relu': 0.1, 'last_activation': 'linear', # sigmoid or linear 'loss': 'mean_square_error', # binary_crossentropy or mean_square_error 'epsilon_std': 1.0, 'batch_size': 20, 'epochs': 100, 'steps_per_epoch': 500 } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() # 2. train/load variational autoencoder vae = VariationalAutoencoder(ae_params) vae.train(all_training_scaled, sets_test_scaled) vae.save_model("vae_" + ae_params_hash) # vae.load_model("vae_" + ae_params_hash) # 3: encode data using autoencoder sets_encoded_training = [] for set_training_scaled in sets_training_scaled: sets_encoded_training.append(vae.encode(set_training_scaled)) sets_encoded_test = [] for set_test_scaled in sets_test_scaled: sets_encoded_test.append(vae.encode(set_test_scaled)) # 4: decode using vae decoded_data = vae.decode(sets_encoded_test[0]) # 7: undo minimax, for now only the first simulation simulated = preprocess_normalisation.rescale_data( decoded_data, dataset_name=test_dataset_names[0]) # reconstruction error # reconstruction_error(sets_test_scaled[0], decoded_data) reconstruction_error(np.array(sets_test[0]), simulated) # plot latent space plotting.plot_2d(sets_encoded_test[0], "test_feature_normalised_encoded_vae_on_", save=True) plotting.plot_space(maturities, vae, "variational_grid", latent_dim=sets_encoded_test[0].shape[1]) # plot scaled results plotting.plot_some_curves("test_feature_normalised_compare_vae_scaled", sets_test_scaled[0], decoded_data, [25, 50, 75, 815], maturities) plotting.plot_some_curves("test_feature_normalised_compare_vae", sets_test[0], simulated, [25, 50, 75, 815], maturities)
def simulate(plot=True): plotting = Plotting() preprocess = PreprocessData() preprocess.enable_normalisation_scaler = True preprocess.feature_range = [0, 1] window_size = 20 # 1. get data and apply normalisation sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess.get_data( chunks_of=window_size) print("sets_training_scaled.shape", sets_training_scaled[0].shape) # plotting.plot_2d(sets_training_scaled[0][:, 0], "sets_training_scaled[0][:, 0]", save=False) # plotting.plot_2d(sets_test_scaled[0][:, 0], "test_feature_normalised_short_end", save=True) ae_params = { 'input_dim': ( window_size, sets_training_scaled[0].shape[1], ), # 10 x 56 'latent_dim': ( 2, 56, ), 'hidden_layers': ( 12 * 56, 4 * 56, ), 'leaky_relu': 0.1, 'loss': 'mse', 'last_activation': 'linear', 'batch_size': 20, 'epochs': 100, 'steps_per_epoch': 500, } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = AutoencoderWindows(ae_params) print("sets_training_scaled", sets_training_scaled[0].shape) autoencoder.train(sets_training_scaled, sets_test_scaled) autoencoder.save_model("ae_" + ae_params_hash) # autoencoder.load_model("ae_" + ae_params_hash) # 2: encode data using autoencoder sets_encoded_training = [] for set_training_scaled in sets_training_scaled: sets_encoded_training.append(autoencoder.encode(set_training_scaled)) sets_encoded_test = [] for set_test_scaled in sets_test_scaled: sets_encoded_test.append(autoencoder.encode(set_test_scaled)) print("sets_encoded_training", len(sets_encoded_training), sets_encoded_training[0].shape) print("sets_encoded_test", sets_encoded_test[0].shape) # 6: decode using autoencoder decoded_test = autoencoder.decode(sets_encoded_test[0]) print("decoded_test", decoded_test.shape) # 7: undo minimax, for now only the first simulation # decoded_generated_segments_first_sim = decoded_generated_segments[0] preprocess.enable_curve_smoothing = True simulated_smooth = preprocess.rescale_data( decoded_test, dataset_name=test_dataset_names[0]) # reconstruction error # reconstruction_error(sets_test_scaled[0], decoded_test) # error = reconstruction_error(np.array(sets_test[0]), simulated_smooth) # print("error:", error) smape_result_smooth = smape(simulated_smooth, np.array(sets_test[0]), over_curves=True) print(np.mean(smape_result_smooth), np.var(smape_result_smooth)) if plot: # plotting.plot_2d(sets_encoded_test[0], "test_feature_normalised_encoded_autoencoder_on_", save=True) # plotting.plot_some_curves("normalised_compare_ae_before_rescale", sets_test_scaled[0], decoded_test, # [25, 50, 75, 815], maturities) plotting.plot_some_curves("normalised_compare_ae", sets_test[0], simulated_smooth, [25, 50, 75, 815], maturities)
def simulate(latent_dim=2, plot=False, preprocess_type=None, model_type=None, force_training=True): plotting = Plotting() preprocess = PreprocessData(preprocess_type) window_size = None if model_type is AEModel.AE_WINDOWS: window_size = 10 # 1. get data and apply normalisation sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess.get_data( chunks_of=window_size) all_training_scaled = np.vstack(sets_training_scaled) if model_type is AEModel.AAE: ae_params = { 'preprocess_type': preprocess_type. value, # only to make preprocess_type part of the hash 'input_dim': sets_training_scaled[0].shape[1], # 56 'latent_dim': latent_dim, 'hidden_layers': ( 56, 40, 28, 12, 4, ), 'hidden_layers_discriminator': ( 2, 2, ), 'leaky_relu': 0.1, 'last_activation': 'linear', 'last_activation_discriminator': 'sigmoid', 'loss_generator': 'mean_squared_error', 'loss_discriminator': 'binary_crossentropy', 'batch_size': 20, 'epochs': 20000 } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() # 2. train/load variational autoencoder autoencoder = AdversarialAutoencoder(ae_params, plot=False) elif model_type is AEModel.VAE: ae_params = { 'preprocess_type': preprocess_type. value, # only to make preprocess_type part of the hash 'input_dim': sets_training_scaled[0].shape[1], # 56 'latent_dim': latent_dim, 'hidden_layers': ( 56, 40, 28, 12, 4, ), 'leaky_relu': 0.1, 'last_activation': 'linear', # sigmoid or linear 'loss': 'mean_squared_error', # binary_crossentropy or mean_square_error 'epsilon_std': 1.0, 'batch_size': 20, 'epochs': 100, 'steps_per_epoch': 500 } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() # 2. train/load variational autoencoder autoencoder = VariationalAutoencoder(ae_params, plot=False) elif model_type is AEModel.AE: ae_params = { 'preprocess_type': preprocess_type. value, # only to make preprocess_type part of the hash 'input_dim': sets_training_scaled[0].shape[1], # 56 'latent_dim': latent_dim, 'hidden_layers': ( 56, 40, 28, 12, 4, ), 'leaky_relu': 0.1, 'loss': 'mse', 'last_activation': 'linear', 'batch_size': 20, 'epochs': 100, 'steps_per_epoch': 500 } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = Autoencoder(ae_params, plot=False) elif model_type is AEModel.PCA: ae_params = { 'preprocess_type': preprocess_type. value, # only to make preprocess_type part of the hash 'latent_dim': latent_dim } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = PCAModel(ae_params, plot=False) else: # model_type is AEModel.AE_WINDOWS: ae_params = { 'input_dim': ( window_size, sets_training_scaled[0].shape[1], ), # 10 x 56 'latent_dim': ( 2, 56, ), 'hidden_layers': ( 12 * 56, 4 * 56, ), 'leaky_relu': 0.1, 'loss': 'mse', 'last_activation': 'linear', 'batch_size': 20, 'epochs': 10, 'steps_per_epoch': 500, } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = AutoencoderWindows(ae_params, plot=False) if force_training: autoencoder.train(all_training_scaled, sets_test_scaled, "ae_" + ae_params_hash) else: autoencoder.load_else_train(all_training_scaled, sets_test_scaled, "ae_" + ae_params_hash) # 2: encode data using autoencoder sets_encoded_training = autoencoder.encode(sets_training_scaled) sets_encoded_test = autoencoder.encode(sets_test_scaled) # 6: decode using autoencoder decoded_test = autoencoder.decode(sets_encoded_test[0]) # 7: undo scaling # decoded_generated_segments_first_sim = decoded_generated_segments[0] simulated = preprocess.rescale_data(decoded_test, dataset_name=test_dataset_names[0]) preprocess.enable_curve_smoothing = True simulated_smooth = preprocess.rescale_data( decoded_test, dataset_name=test_dataset_names[0]) # reconstruction error # error = reconstruction_error(np.array(sets_test[0]), simulated) # error_smooth = reconstruction_error(np.array(sets_test[0]), simulated_smooth) smape_result = smape(simulated, np.array(sets_test[0]), over_curves=True) smape_result_smooth = smape(simulated_smooth, np.array(sets_test[0]), over_curves=True) print(np.mean(smape_result_smooth)) if plot and model_type is not AEModel.AE_WINDOWS: plotting.plot_2d(sets_encoded_test[0], preprocess_type.name + "_" + model_type.name + "_latent_space", sets_test_scaled[0].index.values, save=True) plotting.plot_some_curves( preprocess_type.name + "_" + model_type.name + "_in_vs_out", sets_test[0], simulated, [25, 50, 75, 815], maturities) # plotting.plot_some_curves("normalised_compare_ae", sets_test[0], sets_test_scaled[0], # [25, 50, 75, 815, 100, 600, 720, 740], maturities, plot_separate=True) preprocess.enable_curve_smoothing = False if model_type is AEModel.VAE: plotting.plot_grid_2dim(maturities, autoencoder.generator_model, preprocess_type.name + "_" + model_type.name + "_latent_grid", preprocess, test_dataset_names[0], n=6) elif model_type is AEModel.AAE: plotting.plot_grid_2dim(maturities, autoencoder.decoder, preprocess_type.name + "_" + model_type.name + "_latent_grid", preprocess, test_dataset_names[0], n=6) return smape_result_smooth
def simulate(): plotting = Plotting() preprocess_logreturns = PreprocessData() preprocess_logreturns.enable_log_returns = True # 1. get data and apply minimax sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess_logreturns.get_data( ) sets_training_first_last_tenors = [] for set_training_scaled in sets_training_scaled: sets_training_first_last_tenors.append( set_training_scaled.iloc[:, [0, -1]]) # sets_training_first_last_tenors = np.array(sets_training_first_last_tenors) sets_test_first_last_tenors = [] for set_test_scaled in sets_test_scaled: sets_test_first_last_tenors.append(set_test_scaled.iloc[:, [0, -1]]) # sets_test_first_last_tenors = np.array(sets_test_first_last_tenors) data_train = np.vstack(sets_training_first_last_tenors) mask = sample_mask(data_train.shape[0], data_train.shape[1], 0.2) data_masked = data_train * mask + ( 1 - mask) * -1000 # missing values have representation -1000 gan_params = { 'num_tenors': sets_training_first_last_tenors[0].shape[1], 'num_c': 6 * 7, 'num_z': 6 * 7, 'num_o': 6 * 7, 'gen_model_type': 'standard', # conv 'dis_model_type': 'standard', # conv 'gen_layers': (4 * (6 * 7 * 2), ), # 4 * num_o * num_tenors 'dis_layers': (4 * (6 * 7), ), # 4 * num_o 'gen_last_activation': 'tanh', 'dis_last_activation': 'sigmoid', 'loss': 'binary_crossentropy', 'batch_size': 128, 'epochs': 20000, 'mask_p': 0.2 } gan_params_hash = hashlib.md5( json.dumps(gan_params, sort_keys=True).encode('utf-8')).hexdigest() gan = GANMissing(gan_params) gan.train(data_masked, mask) gan.save_model("gan_test_" + gan_params_hash) # gan.load_model("gan_test_" + gan_params_hash) # 4: simulate on encoded log returns, conditioned on test dataset def simulate_with_gan(num_runs=20): generated_segments, real_segment = gan.generate( data=sets_test_first_last_tenors[-1], num_simulations=100, remove_condition=False) last_generated_segment = generated_segments for _ in np.arange(num_runs - 1): generated_temp, real_temp = gan.generate( condition=last_generated_segment, remove_condition=True) last_generated_segment = generated_temp generated_segments = np.append(generated_segments, generated_temp, axis=1) # 5: undo log-returns generated_segments = preprocess_logreturns.rescale_data( generated_segments, start_value=sets_test[-1][-1]) # plotting.plot_3d_many(file_name, data, save=False) plotting.plot_3d_training("3d recursively generated with GAN, test", generated_segments, sets_test[-1], show=True, after_real_data=True) for _ in np.arange(20): simulate_with_gan(num_runs=20)
def simulate(latent_dim=2, preprocess_type1=None, preprocess_type2=None, ae_model=None, gan_model=None, force_training=True, plot=False): preprocess1 = PreprocessData(preprocess_type1) preprocess2 = PreprocessData(preprocess_type2) # 1. get data and apply scaling sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess1.get_data( ) if ae_model is AEModel.AAE: ae_params = { 'preprocess_type': preprocess_type1. value, # only to make preprocess_type part of the hash 'input_dim': sets_training_scaled[0].shape[1], # 56 'latent_dim': latent_dim, 'hidden_layers': ( 56, 40, 28, 12, 4, ), 'hidden_layers_discriminator': ( 2, 2, ), 'leaky_relu': 0.1, 'last_activation': 'linear', 'last_activation_discriminator': 'sigmoid', 'loss_generator': 'mean_squared_error', 'loss_discriminator': 'binary_crossentropy', 'batch_size': 20, 'epochs': 20000 } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = AdversarialAutoencoder(ae_params, plot=False) elif ae_model is AEModel.VAE: ae_params = { 'preprocess_type': preprocess_type1. value, # only to make preprocess_type part of the hash 'input_dim': sets_training_scaled[0].shape[1], # 56 'latent_dim': latent_dim, 'hidden_layers': ( 56, 40, 28, 12, 4, ), 'leaky_relu': 0.1, 'last_activation': 'linear', # sigmoid or linear 'loss': 'mean_square_error', # binary_crossentropy or mean_square_error 'epsilon_std': 1.0, 'batch_size': 20, 'epochs': 100, 'steps_per_epoch': 500 } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = VariationalAutoencoder(ae_params, plot=False) elif ae_model is AEModel.AE: ae_params = { 'preprocess_type': preprocess_type1. value, # only to make preprocess_type part of the hash 'input_dim': sets_training_scaled[0].shape[1], # 56 'latent_dim': latent_dim, 'hidden_layers': ( 56, 40, 28, 12, 4, ), 'leaky_relu': 0.1, 'loss': 'mse', 'last_activation': 'linear', 'batch_size': 20, 'epochs': 100, 'steps_per_epoch': 500 } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = Autoencoder(ae_params, plot=False) else: # elif ae_model is AEModel.PCA: ae_params = { 'preprocess_type': preprocess_type1. value, # only to make preprocess_type part of the hash 'latent_dim': latent_dim } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = PCAModel(ae_params, plot=False) # 2. train/load autoencoder autoencoder.load_else_train(np.vstack(sets_training_scaled), sets_test_scaled, "ae_" + ae_params_hash) # 2: encode data using autoencoder sets_encoded_training = autoencoder.encode(sets_training_scaled) sets_encoded_test = autoencoder.encode(sets_test_scaled) # 3: log returns of encoded data sets_encoded_log_training = preprocess2.scale_data(sets_encoded_training, training_dataset_names, should_fit=True) sets_encoded_log_test = preprocess2.scale_data(sets_encoded_test, test_dataset_names, should_fit=True) num_z = 6 * 7 num_c = 6 * 7 num_o = 6 * 7 if gan_model is GANModel.WGAN: gan_params = { 'ae_params_hash': ae_params_hash, 'num_tenors': sets_encoded_log_training[0].shape[1], 'num_c': num_c, 'num_z': num_z, 'num_o': num_o, 'gen_model_type': 'standard', # conv 'dis_model_type': 'standard', # conv 'gen_layers': (4 * (6 * 7 * 2), ), # 4 * num_o * num_tenors 'dis_layers': (4 * (6 * 7), ), # 4 * num_o 'gen_last_activation': 'tanh', 'dis_last_activation': 'sigmoid', 'loss': 'binary_crossentropy', 'batch_size': 32, 'epochs': 10000, 'sample_interval': 1000 } gan_params_hash = hashlib.md5( json.dumps(gan_params, sort_keys=True).encode('utf-8')).hexdigest() gan = CWGANGP(gan_params, plot=False) else: if gan_model is GANModel.GAN_CONV: model_type = 'conv' else: # if gan_model is GANModel.GAN: model_type = 'standard' gan_params = { 'ae_params_hash': ae_params_hash, 'num_tenors': sets_encoded_log_training[0].shape[1], 'num_c': num_c, 'num_z': num_z, 'num_o': num_o, 'gen_model_type': model_type, # conv 'dis_model_type': model_type, # conv 'gen_layers': (4 * (6 * 7 * 2), ), # 4 * num_o * num_tenors 'dis_layers': (4 * (6 * 7), ), # 4 * num_o 'gen_last_activation': 'tanh', 'dis_last_activation': 'sigmoid', 'loss': 'binary_crossentropy', 'batch_size': 128, 'epochs': 20000 } gan_params_hash = hashlib.md5( json.dumps(gan_params, sort_keys=True).encode('utf-8')).hexdigest() gan = GAN(gan_params, plot=False) # try training on larger input and output if force_training: gan.train(sets_encoded_log_training, "gan_" + gan_params_hash) else: gan.load_else_train(sets_encoded_log_training, "gan_" + gan_params_hash) # 4: simulate on encoded log returns, conditioned on test dataset num_simulations = 100 num_repeats = 1 generated, _ = gan.generate(condition=sets_encoded_log_test[-1], condition_on_end=False, num_simulations=num_simulations, repeat=num_repeats) # insert the last real futures curve in order to do rescaling if preprocess_type2 is PreprocessType.LOG_RETURNS_OVER_TENORS: generated = np.insert(generated, 0, sets_encoded_log_test[-1].iloc[num_c], axis=1) # 5: undo scaling encoded_generated = preprocess2.rescale_data( generated, start_value=sets_encoded_test[-1][num_c], dataset_name=test_dataset_names[-1]) if preprocess_type2 is PreprocessType.LOG_RETURNS_OVER_TENORS: encoded_generated = encoded_generated[:, 1:] # remove first curve again # 6: decode using autoencoder decoded_generated_segments = autoencoder.decode(encoded_generated) # 7: undo scaling, this can be log-returns simulated = preprocess1.rescale_data(decoded_generated_segments, start_value=sets_test[-1].iloc[num_c], dataset_name=test_dataset_names[-1]) preprocess1.enable_curve_smoothing = True simulated_smooth = preprocess1.rescale_data( decoded_generated_segments, start_value=sets_test[-1].iloc[num_c], dataset_name=test_dataset_names[-1]) if preprocess_type2 is PreprocessType.LOG_RETURNS_OVER_TENORS: real = sets_test[-1].iloc[ num_c:num_c + num_o * num_repeats + 1] # `+1` because the log-returns also does +1 else: real = sets_test[-1].iloc[num_c:num_c + num_o * num_repeats + 1] print("simulated, real", simulated.shape, real.shape) smape_result = smape(simulated, real) smape_result_smooth = smape(simulated_smooth, real) print("smape_result_smooth mean and std:", np.mean(smape_result_smooth), np.std(smape_result_smooth)) if plot: plotting = Plotting() plotting.plot_3d("real", real, show_title=False) cov_log_returns = cov_log_returns_over_tenors(real) plotting.plot_3d_cov("gan_real_cov", cov_log_returns, show_title=False) for i in np.arange(1, 11): # name = '_' + preprocess_type1.name + '_' + preprocess_type2.name + '_' + str(latent_dim) + '_' + ae_model.name + '_'+ gan_model.name plotting.plot_3d("gan_simulated_" + str(i), simulated_smooth[i], maturities=maturities, time=real.index.values, show_title=False) smape_result = smape(simulated_smooth[i], real) print("simulated_smooth[i], real", simulated_smooth[i].shape, real.shape) print("simulate rates", i) print("smape:", smape_result) print("=============\n") cov_log_returns = cov_log_returns_over_tenors(simulated_smooth[i]) plotting.plot_3d_cov("gan_simulated_" + str(i) + "_cov", cov_log_returns, maturities=maturities, show_title=False) return smape_result_smooth
def simulate(): plotting = Plotting() preprocess_normalisation = PreprocessData() preprocess_normalisation.enable_normalisation_scaler = True preprocess_normalisation.feature_range = [0, 1] # preprocess_normalisation.enable_scaler = True # 1. get data and apply normalisation sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess_normalisation.get_data( ) # plotting.plot_2d(sets_training_scaled[0][:, 0], "sets_training_scaled[0][:, 0]", save=False) # plotting.plot_2d(sets_test_scaled[0][:, 0], "test_feature_normalised_short_end", save=True) all_stacked = np.vstack((np.vstack(sets_training), np.vstack(sets_test))) all_stacked_scaled = np.vstack( (np.vstack(sets_training_scaled), np.vstack(sets_test_scaled))) all_training_scaled = np.vstack(sets_training_scaled) # print("all_stacked_scaled.shape", all_stacked_scaled.shape) # plotting.plot_2d(all_stacked[:, 0], "training and test data", save=False) # plotting.plot_2d(all_stacked_scaled[:, 0], "training and test data scaled", save=False) ae_params = { 'input_dim': sets_training_scaled[0].shape[1], # 56 'latent_dim': 2, 'hidden_layers': (56, 40, 28, 12, 4, 2), 'leaky_relu': 0.1, 'loss': 'mse', 'last_activation': 'linear', 'batch_size': 20, 'epochs': 100, 'steps_per_epoch': 500 } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = Autoencoder(ae_params) # autoencoder.train(all_stacked_scaled, sets_test_scaled) # autoencoder.train(sets_test_scaled[0], sets_test_scaled) # autoencoder.train(all_training_scaled, sets_test_scaled) # autoencoder.save_model("ae_" + ae_params_hash) autoencoder.load_model("ae_" + ae_params_hash) # 2: encode data using autoencoder sets_encoded_training = [] for set_training_scaled in sets_training_scaled: sets_encoded_training.append(autoencoder.encode(set_training_scaled)) sets_encoded_test = [] for set_test_scaled in sets_test_scaled: sets_encoded_test.append(autoencoder.encode(set_test_scaled)) plotting.plot_2d(sets_encoded_test[0], "test_feature_normalised_encoded_autoencoder_on_", save=True) # 6: decode using autoencoder decoded_test = autoencoder.decode(sets_encoded_test[0]) # 7: undo minimax, for now only the first simulation simulated = preprocess_normalisation.rescale_data( decoded_test, dataset_name=test_dataset_names[0]) plotting.plot_some_curves( "test_feature_normalised_compare_autoencoder_before_rescale", sets_test_scaled[0], decoded_test, [25, 50, 75, 815], maturities) # old: [25, 50, 75, 100, 600, 720, 740, 815] plotting.plot_some_curves( "test_feature_normalised_compare_autoencoder", sets_test[0], simulated, [25, 50, 75, 815], maturities) # old: [25, 50, 75, 100, 600, 720, 740, 815] # curve_smooth = [] # for curve in simulated: # print("curve.shape", curve.shape) # curve_smooth.append(savgol_filter(curve, 23, 5)) # window size 51, polynomial order 3 # curve_smooth = np.array(curve_smooth) print("reconstruction error BEFORE smoothing:") reconstruction_error(np.array(sets_test[0]), simulated) preprocess_normalisation.enable_curve_smoothing = True simulated = preprocess_normalisation.rescale_data( decoded_test, dataset_name=test_dataset_names[0]) plotting.plot_some_curves( "test_feature_normalised_compare_autoencoder", sets_test[0], simulated, [25, 50, 75, 815], maturities) # old: [25, 50, 75, 100, 600, 720, 740, 815] # plotting.plot_some_curves("test_feature_normalised_compare_normalisation", sets_test[0], sets_test_scaled[0], # [25, 50, 75, 815, 100, 600, 720, 740], maturities, plot_separate=True) # reconstruction error # reconstruction_error(sets_test_scaled[0], decoded_test) print("reconstruction error AFTER smoothing:") reconstruction_error(np.array(sets_test[0]), simulated)
def simulate(): plotting = Plotting() preprocess_type = PreprocessType.STANDARDISATION_OVER_TENORS preprocess = PreprocessData(preprocess_type) # 1. get data and apply minimax sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess.get_data( ) all_training_scaled = np.vstack(sets_training_scaled) ae_params = { 'preprocess_type': preprocess_type.value, # only to make preprocess_type part of the hash 'input_dim': sets_training_scaled[0].shape[1], # 56 'latent_dim': 2, 'hidden_layers': ( 56, 40, 28, 12, 4, ), 'leaky_relu': 0.1, 'loss': 'mse', 'last_activation': 'linear', 'batch_size': 20, 'epochs': 100, 'steps_per_epoch': 500 } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = Autoencoder(ae_params) autoencoder.load_else_train(all_training_scaled, sets_test_scaled, "ae_" + ae_params_hash) # 2: encode data using autoencoder encoded = autoencoder.encode(sets_test_scaled[0]) decoded = autoencoder.decode(encoded) rescaled = preprocess.rescale_data(decoded, dataset_name=test_dataset_names[0]) smape_result = smape(rescaled, np.array(sets_test[0]), over_curves=True) print("smape_result test set", np.mean(smape_result), np.std(smape_result), np.min(smape_result), np.max(smape_result)) plotting.plot_2d(sets_test[0], "evaluation of test curves", timeseries=True, evaluation=smape_result, title=False) # for i in np.arange(len(test_eval)): # if test_eval[i] > 4: # plotting.plot_2d(sets_test_scaled[0][i], "Possible unrealistic curve" + str(i), save=False, title=True) # 3: lets see how well the autoencoder can map a zero vector # todo: generate random curves, THEN apply min-max feature scaling, THEN evaluate unrealistic_curves = [] curve_shape = 56 unrealistic_curves.append(np.full(curve_shape, 5)) unrealistic_curves.append(np.full(curve_shape, 10)) unrealistic_curves.append(np.full(curve_shape, 20)) unrealistic_curves.append(np.full(curve_shape, 50)) unrealistic_curves.append(np.full(curve_shape, 70)) unrealistic_curves.append(np.full(curve_shape, 100)) unrealistic_curves.append(np.full(curve_shape, 150)) unrealistic_curves.append(np.full(curve_shape, 200)) unrealistic_curves.append(np.full(curve_shape, 250)) unrealistic_curves.append(np.full(curve_shape, 300)) unrealistic_curves.append( np.hstack((np.full(int(curve_shape / 2), 50), np.full(int(curve_shape / 2), 150)))) unrealistic_curves.append( np.hstack((np.full(int(curve_shape / 2), 100), np.full(int(curve_shape / 2), 150)))) unrealistic_curves.append( np.hstack((np.full(int(curve_shape / 2), 100), np.full(int(curve_shape / 2), 200)))) unrealistic_curves.append(np.random.uniform(0, 10, curve_shape)) unrealistic_curves.append(np.random.uniform(10, 70, curve_shape)) unrealistic_curves.append(np.random.uniform(0, 100, curve_shape)) unrealistic_curves.append(np.random.uniform(100, 200, curve_shape)) unrealistic_curves.append(np.random.uniform(200, 300, curve_shape)) unrealistic_curves.append(np.random.uniform(0, 200, curve_shape)) unrealistic_curves.append(np.random.uniform(0, 250, curve_shape)) unrealistic_curves.append(np.random.uniform(0, 300, curve_shape)) unrealistic_curves.append(np.linspace(0, 100, num=curve_shape)) unrealistic_curves.append(np.linspace(50, 150, num=curve_shape)) unrealistic_curves.append(np.linspace(100, 200, num=curve_shape)) unrealistic_curves.append(np.linspace(150, 250, num=curve_shape)) unrealistic_curves.append(np.linspace(200, 300, num=curve_shape)) unrealistic_curves.append(np.linspace(0, 200, num=curve_shape)) unrealistic_curves.append(np.linspace(0, 300, num=curve_shape)) unrealistic_curves.append(np.linspace(100, 0, num=curve_shape)) unrealistic_curves.append(np.linspace(150, 50, num=curve_shape)) unrealistic_curves.append(np.linspace(200, 100, num=curve_shape)) unrealistic_curves.append(np.linspace(250, 150, num=curve_shape)) unrealistic_curves.append(np.linspace(300, 200, num=curve_shape)) unrealistic_curves.append(np.linspace(200, 0, num=curve_shape)) unrealistic_curves.append(np.linspace(300, 0, num=curve_shape)) unrealistic_curves = np.array(unrealistic_curves) print("unrealistic_curves.shape", unrealistic_curves.shape) unrealistic_curves_scaled = preprocess.scale_data( unrealistic_curves, dataset_name=training_dataset_names[0], should_fit=True) encoded = autoencoder.encode(unrealistic_curves_scaled) decoded = autoencoder.decode(encoded) rescaled = preprocess.rescale_data(decoded, dataset_name=training_dataset_names[0]) smape_result = smape(rescaled, unrealistic_curves, over_curves=True) round_to_n = lambda x, n: round(x, -int(np.floor(np.log10(x))) + (n - 1)) print("smape results", smape_result) for a_smape_result in smape_result: print(round_to_n(a_smape_result, 2)) plotting.plot_2d(smape_result, "loss of unrealistic curves from autoencoder SMAPE", save=False, title=True) plotting.plot_2d(smape_result, "loss of unrealistic curves from autoencoder SMAPE", save=False, title=True) # plotting.plot_2d(unrealistic_eval_mse, "loss of unrealistic curves from autoencoder MSE", save=False, title=True) plotting.plot_unrealisticness( unrealistic_curves, "loss of unrealistic curves from autoencoder", timeseries=True, evaluation=smape_result, title=False, eval_label="SMAPE")
def simulate(): plotting = Plotting() preprocess_minmax = PreprocessData() preprocess_logreturns = PreprocessData() preprocess_minmax.enable_min_max_scaler = True preprocess_logreturns.enable_log_returns = True # 1. get data and apply minimax sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess_minmax.get_data( ) print("sets_training_scaled.shape", sets_training_scaled[0].shape) autoencoder = DeepAutoencoder( input_shape=(sets_training_scaled[0].shape[1], ), latent_dim=2) # autoencoder.train(np.vstack(sets_training_scaled), sets_test_scaled, epochs=100, batch_size=5) # autoencoder.save_model("deep_general_minimax") autoencoder.load_model("deep_general_minimax") # 2: encode data using autoencoder sets_encoded_training = [] for set_training_scaled in sets_training_scaled: sets_encoded_training.append(autoencoder.encode(set_training_scaled)) sets_encoded_test = [] for set_test_scaled in sets_test_scaled: sets_encoded_test.append(autoencoder.encode(set_test_scaled)) plotting.plot_2d(sets_encoded_test[0], "encoded test data with deep autoencoder", save=False) # 3: log returns of encoded data sets_encoded_log_training = [] for index, set_encoded_training in enumerate(sets_encoded_training): sets_encoded_log_training.append( preprocess_logreturns.scale_data(set_encoded_training)) sets_encoded_log_test = [] for index, set_encoded_test in enumerate(sets_encoded_test): sets_encoded_log_test.append( preprocess_logreturns.scale_data(set_encoded_test)) plotting.plot_2d( sets_encoded_log_test[0], "encoded test data with deep autoencoder, then log returns", save=False) num_tenors = sets_encoded_log_training[0].shape[1] gan = GAN(num_c=6 * 7, num_z=6 * 7, num_o=6 * 7, num_tenors=num_tenors) # try training on larger input and output # gan.train(sets_encoded_log_training, epochs=20000, batch_size=100, sample_interval=200) # gan.save_model("general_ae") gan.load_model("general_ae") print("sets_encoded_log_test[0].shape", sets_encoded_log_test[0].shape) test_arr = np.full([1, 6 * 7 + 6 * 7, num_tenors], 10) validity = gan.discriminator.predict( test_arr) # np.array(sets_encoded_log_test[0] print(validity) rolled_encoded_log_test = rolling_windows(sets_encoded_log_test[0], 6 * 7 + 6 * 7) validity = gan.discriminator.predict( rolled_encoded_log_test) # np.array(sets_encoded_log_test[0] print(validity)
def simulate(latent_dim=2, preprocess_type1=None, preprocess_type2=None, ae_model=None, gan_model=None, force_training=True, plot=False): preprocess1 = PreprocessData(preprocess_type1, short_end=True) preprocess2 = PreprocessData(preprocess_type2, short_end=True) # 1. get data and apply scaling sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess1.get_data( ) print("sets_test_scaled, sets_training_scaled:", sets_test_scaled[0].shape, sets_training_scaled[0].shape) # 2: log returns of encoded data sets_encoded_log_training = preprocess2.scale_data(sets_training_scaled, training_dataset_names, should_fit=True) sets_encoded_log_test = preprocess2.scale_data(sets_test_scaled, test_dataset_names, should_fit=True) num_c = 6 * 7 num_o = 6 * 7 if gan_model is GANModel.WGAN: gan_params = { 'short_end_encoding': preprocess_type1.name + "_" + preprocess_type2.name, 'num_tenors': sets_encoded_log_training[0].shape[1], 'num_c': 6 * 7, 'num_z': 6 * 7, 'num_o': 6 * 7, 'gen_model_type': 'standard', # conv 'dis_model_type': 'standard', # conv 'gen_layers': (4 * (6 * 7 * 2), ), # 4 * num_o * num_tenors 'dis_layers': (4 * (6 * 7), ), # 4 * num_o 'gen_last_activation': 'tanh', 'dis_last_activation': 'sigmoid', 'loss': 'binary_crossentropy', 'batch_size': 32, 'epochs': 10000, 'sample_interval': 1000 } gan_params_hash = hashlib.md5( json.dumps(gan_params, sort_keys=True).encode('utf-8')).hexdigest() gan = CWGANGP(gan_params, plot=False) else: if gan_model is GANModel.GAN_CONV: model_type = 'conv' else: # if gan_model is GANModel.GAN: model_type = 'standard' print("num tenors:", sets_encoded_log_training[0].shape[1]) gan_params = { 'short_end_encoding': preprocess_type1.name + "_" + preprocess_type2.name, 'num_tenors': sets_encoded_log_training[0].shape[1], 'num_c': num_c, 'num_z': 6 * 7, 'num_o': num_o, 'gen_model_type': model_type, # conv 'dis_model_type': model_type, # conv 'gen_layers': (4 * (6 * 7 * 2), ), # 4 * num_o * num_tenors 'dis_layers': (4 * (6 * 7), ), # 4 * num_o 'gen_last_activation': 'tanh', 'dis_last_activation': 'sigmoid', 'loss': 'binary_crossentropy', 'batch_size': 128, 'epochs': 20000 } gan_params_hash = hashlib.md5( json.dumps(gan_params, sort_keys=True).encode('utf-8')).hexdigest() gan = GAN(gan_params, plot=False) # try training on larger input and output if force_training: gan.train(sets_encoded_log_training, "gan_" + gan_params_hash) else: gan.load_else_train(sets_encoded_log_training, "gan_" + gan_params_hash) # 4: simulate on encoded log returns, conditioned on test dataset num_simulations = 100 num_repeats = 0 print("sets_encoded_log_test[-1]", sets_encoded_log_test[-1].shape) generated, _ = gan.generate(condition=sets_encoded_log_test[-1], condition_on_end=False, num_simulations=num_simulations, repeat=num_repeats) # insert the last real futures curve in order to do rescaling if preprocess_type2 is PreprocessType.LOG_RETURNS_OVER_TENORS: generated = np.insert(generated, 0, sets_encoded_log_test[-1].iloc[num_c], axis=1) print("sets_test_scaled[-1]", sets_test_scaled[-1].shape) print("sets_test_scaled[-1][num_c]", sets_test_scaled[-1].iloc[num_c]) # 5: undo scaling encoded_generated = preprocess2.rescale_data( generated, start_value=sets_test_scaled[-1].iloc[num_c], dataset_name=test_dataset_names[-1]) if preprocess_type2 is PreprocessType.LOG_RETURNS_OVER_TENORS: encoded_generated = encoded_generated[:, 1:] # remove first curve again # 7: undo scaling, this can be log-returns simulated = preprocess1.rescale_data(encoded_generated, start_value=sets_test[-1].iloc[num_c], dataset_name=test_dataset_names[-1]) if preprocess_type2 is PreprocessType.LOG_RETURNS_OVER_TENORS: real = np.array( sets_test[-1])[num_c:num_c + num_o + 1] # `+1` because the log-returns also does +1 else: real = np.array(sets_test[-1])[num_c:num_c + num_o + 1] sim = simulated.reshape(100, 43) print("sets_test[-1].iloc[num_c], sim[0][0]", sets_test[-1].iloc[num_c], sim[0][0], sim[1][0], sim[2][0]) print("real, simulated", real.shape, sim.shape) smape_result = smape(sim, real, over_curves=True) if plot: condition_and_real = sets_test[-1].iloc[0:num_c + num_o + 1] plotting = Plotting() plotting.plot_training_sample("simulated_simple", sim, condition_and_real, num_c, after_real_data=True) # print("smape test:", smape(simulated[0], real), smape_result) return smape_result
def simulate(): plotting = Plotting() preprocess_normalisation = PreprocessData() preprocess_logreturns = PreprocessData() preprocess_normalisation.enable_normalisation_scaler = True preprocess_logreturns.enable_log_returns = True # 1. get data and apply pre-processing sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess_normalisation.get_data() ae_params = { 'preprocess_type': PreprocessType.NORMALISATION_OVER_TENORS.value, 'input_dim': (10, sets_training_scaled[0].shape[1],), # 56 'latent_dim': 2*56, 'hidden_layers': (12*56, 4*56, ), 'leaky_relu': 0.1, 'loss': 'mse', 'last_activation': 'linear', 'batch_size': 5, 'epochs': 5, 'steps_per_epoch': 500} ae_params_hash = hashlib.md5(json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = Autoencoder(ae_params) # autoencoder.train(np.vstack(sets_training_scaled), sets_test_scaled) # autoencoder.save_model("ae_" + ae_params_hash) autoencoder.load_else_train(sets_training_scaled, sets_test_scaled, "ae_" + ae_params_hash) # 2: encode data using autoencoder sets_encoded_training = autoencoder.encode(sets_training_scaled) sets_encoded_test = autoencoder.encode(sets_test_scaled) print("sets_encoded_test", sets_encoded_test[0].shape) plotting.plot_2d(sets_encoded_test[0], "encoded test data with deep autoencoder", save=False) # 3: log returns of encoded data sets_encoded_log_training = preprocess_logreturns.scale_data(sets_encoded_training) sets_encoded_log_test = preprocess_logreturns.scale_data(sets_encoded_test) plotting.plot_2d(sets_encoded_log_test[0], "encoded test data with deep autoencoder, then log returns", save=False) num_c = 6*7 num_o = 6*7 gan_params = {'ae_params_hash': ae_params_hash, 'num_tenors': sets_encoded_log_training[0].shape[1], 'num_c': num_c, 'num_z': 6*7, 'num_o': num_o, 'gen_model_type': 'standard', # conv 'dis_model_type': 'standard', # conv 'gen_layers': (4*(6*7*2),), # 4 * num_o * num_tenors 'dis_layers': (4*(6*7),), # 4 * num_o 'gen_last_activation': 'tanh', 'dis_last_activation': 'sigmoid', 'loss': 'binary_crossentropy', 'batch_size': 128, 'epochs': 20000} gan_params_hash = hashlib.md5(json.dumps(gan_params, sort_keys=True).encode('utf-8')).hexdigest() gan = GAN(gan_params) # try training on larger input and output # gan.train(sets_encoded_log_training, sample_interval=200) # gan.save_model("gan_" + gan_params_hash) gan.load_model("gan_" + gan_params_hash) # COV TEST, TEMPORARY # for name, set in zip(training_dataset_names, sets_training): # print("name:", name) # set_cov_log_returns_over_features = cov_log_returns_over_features(set) # plotting.plot_3d_cov("covariance_time_series_" + name, set_cov_log_returns_over_features, show_title=False) # plotting.plot_3d("time_series_" + name, set, maturities) # END COV TEST. # 4: simulate on encoded log returns, conditioned on test dataset num_simulations = 10 num_repeats = 0 generated, _ = gan.generate(condition=sets_encoded_log_test[-1], condition_on_end=False, num_simulations=num_simulations, repeat=num_repeats) # insert the last real futures curve in order to do rescaling print("sets_encoded_log_test[-1][num_c] shape", sets_encoded_log_test[-1].iloc[num_c].shape) print("generated_segments.shape", generated.shape) generated = np.insert(generated, 0, sets_encoded_log_test[-1].iloc[num_c], axis=0) # 5: undo log-returns # todo: this start_value is actually one off! Error still persists... autoencoder causing the difference? encoded_generated = preprocess_logreturns.rescale_data(generated, start_value=sets_encoded_test[-1][num_c]) encoded_generated = encoded_generated[:, 1:] # remove first curve again # 6: decode using autoencoder decoded_generated_segments = autoencoder.decode(encoded_generated) # 7: undo minimax, for now only the first simulation simulated = preprocess_normalisation.rescale_data(decoded_generated_segments, dataset_name=test_dataset_names[-1]) preprocess_normalisation.enable_curve_smoothing = True simulated_smooth = preprocess_normalisation.rescale_data(decoded_generated_segments, dataset_name=test_dataset_names[-1]) real = np.array(sets_test[-1])[num_c:num_c + num_o] print("simulated, real", simulated.shape, real.shape) smape_result = smape(simulated, real) smape_result_smooth = smape(simulated_smooth, real) print("smape_result and smooth", smape_result, smape_result_smooth) print("smape_resul_smooth", smape_result_smooth)
from helpers.preprocess_data import PreprocessData from helpers.evaluate import * from helpers.plotting import Plotting from imputance.gain_model import gain import numpy as np import matplotlib.pyplot as plt if __name__ == '__main__': plotting = Plotting() preprocess = PreprocessData(PreprocessType.STANDARDISATION_OVER_TENORS, short_end=True) preprocess2 = PreprocessData(PreprocessType.LOG_RETURNS_OVER_TENORS, short_end=True) sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess.get_data( ) sets_encoded_log_training = preprocess2.scale_data(sets_training_scaled, training_dataset_names, should_fit=True) sets_encoded_log_test = preprocess2.scale_data(sets_test_scaled, test_dataset_names, should_fit=True) train = sets_encoded_log_training[0].copy() test = sets_encoded_log_test[0].copy() # print("train.shape[1]", train.shape[1]) # print("sets_test_scaled[0]", sets_test_scaled[0].shape) # print("sets_encoded_log_test[0]", sets_encoded_log_test[0].shape) params = {
def simulate(): plotting = Plotting() preprocess_logreturns = PreprocessData() preprocess_logreturns.enable_log_returns = True # 1. get data and apply minimax sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess_logreturns.get_data( ) sets_training_first_last_tenors = [] for set_training_scaled in sets_training_scaled: sets_training_first_last_tenors.append(set_training_scaled[:, [0, -1]]) # sets_training_first_last_tenors = np.array(sets_training_first_last_tenors) sets_test_first_last_tenors = [] for set_test_scaled in sets_test_scaled: sets_test_first_last_tenors.append(set_test_scaled[:, [0, -1]]) # sets_test_first_last_tenors = np.array(sets_test_first_last_tenors) # scaler = MinMaxScaler() to_train = np.vstack(sets_training_first_last_tenors) to_train = scaler.fit_transform(np.vstack(sets_training_first_last_tenors)) gan_params = { 'num_tenors': sets_training_first_last_tenors[0].shape[1], 'num_c': 6 * 7, 'num_z': 6 * 7, 'num_o': 6 * 7, 'gen_model_type': 'standard', # conv 'dis_model_type': 'standard', # conv 'gen_layers': (4 * (6 * 7 * 2), ), # 4 * num_o * num_tenors 'dis_layers': (4 * (6 * 7), ), # 4 * num_o 'gen_last_activation': 'tanh', 'dis_last_activation': 'sigmoid', 'loss': 'binary_crossentropy', 'batch_size': 32, 'epochs': 10000, 'sample_interval': 1000 } gan_params_hash = hashlib.md5( json.dumps(gan_params, sort_keys=True).encode('utf-8')).hexdigest() # print("min, max", np.min(np.vstack(sets_training_first_last_tenors)), np.max(np.vstack(sets_training_first_last_tenors))) gan = CWGANGP(gan_params) gan.train(to_train) gan.save_model("gwgan_gp_" + gan_params_hash) # gan.load_model("gwgan_gp_" + gan_params_hash) # 4: simulate on encoded log returns, conditioned on test dataset num_simulations = 10 num_repeats = 20 generated_segments, real_segment = gan.generate( data=sets_test_first_last_tenors[-1], num_simulations=num_simulations, remove_condition=False) last_generated_segment = generated_segments for _ in np.arange(num_repeats - 1): generated_temp, real_temp = gan.generate( condition=last_generated_segment, remove_condition=True) last_generated_segment = generated_temp generated_segments = np.append(generated_segments, generated_temp, axis=1) # undo scaler: # temp_generated_segments = [] # for i in np.arange(generated_segments.shape[0]): # temp_generated_segments.append(scaler.inverse_transform(generated_segments[i])) # generated_segments = np.array(temp_generated_segments) # 5: undo log-returns generated_segments = preprocess_logreturns.rescale_data( generated_segments, start_value=sets_test_first_last_tenors[-1][-1]) # plotting.plot_3d_many(file_name, data, save=False) plotting.plot_3d_training("3d recursively generated with GAN, test", generated_segments, sets_test[-1], show=True, after_real_data=True)
def __init__(self, params): self.config = Config() self.plotting = Plotting() self.params = params
def simulate(plot=True): plotting = Plotting() preprocess = PreprocessData() preprocess.enable_normalisation_scaler = True preprocess.feature_range = [0, 1] # 1. get data and apply normalisation sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess.get_data( ) print("sets_training_scaled.shape", sets_training_scaled[0].shape) # plotting.plot_2d(sets_training_scaled[0][:, 0], "sets_training_scaled[0][:, 0]", save=False) # plotting.plot_2d(sets_test_scaled[0][:, 0], "test_feature_normalised_short_end", save=True) ae_params = { 'input_dim': sets_training_scaled[0].shape[1], # 56 'latent_dim': 2, 'hidden_layers': ( 56, 40, 28, 12, 4, ), 'leaky_relu': 0.1, 'loss': 'mse', 'last_activation': 'linear', 'batch_size': 20, 'epochs': 100, 'steps_per_epoch': 500 } ae_params_hash = hashlib.md5( json.dumps(ae_params, sort_keys=True).encode('utf-8')).hexdigest() autoencoder = Autoencoder(ae_params) autoencoder.train(sets_training_scaled, sets_test_scaled) autoencoder.save_model("ae_" + ae_params_hash) # autoencoder.load_model("ae_" + ae_params_hash) # 2: encode data using autoencoder sets_encoded_training = [] for set_training_scaled in sets_training_scaled: sets_encoded_training.append(autoencoder.encode(set_training_scaled)) sets_encoded_test = [] for set_test_scaled in sets_test_scaled: sets_encoded_test.append(autoencoder.encode(set_test_scaled)) # 6: decode using autoencoder decoded_test = autoencoder.decode(sets_encoded_test[0]) # 7: undo minimax, for now only the first simulation # decoded_generated_segments_first_sim = decoded_generated_segments[0] simulated = preprocess.rescale_data(decoded_test, dataset_name=test_dataset_names[0]) # reconstruction error # reconstruction_error(sets_test_scaled[0], decoded_test) error = reconstruction_error(np.array(sets_test[0]), simulated) if plot: plotting.plot_2d(sets_encoded_test[0], "test_feature_normalised_encoded_autoencoder_on_", save=True) plotting.plot_some_curves("normalised_compare_ae_before_rescale", sets_test_scaled[0], decoded_test, [25, 50, 75, 815], maturities) plotting.plot_some_curves("normalised_compare_ae", sets_test[0], simulated, [25, 50, 75, 815], maturities) plotting.plot_some_curves("normalised_compare_ae", sets_test[0], sets_test_scaled[0], [25, 50, 75, 815, 100, 600, 720, 740], maturities, plot_separate=True) return error
def main(): plotting = Plotting() preprocess_normalisation = PreprocessData() preprocess_normalisation.enable_normalisation_scaler = True # preprocess_normalisation.enable_standardisation_scaler = True sets_training, sets_test, sets_training_scaled, sets_test_scaled, training_dataset_names, test_dataset_names, maturities = preprocess_normalisation.get_data( ) # sklearn model (check that it is doing the same (it is)) # pca_model_sklearn = PCA(n_components=2) # pca_model_sklearn.fit(sets_test_scaled[0]) # test_data_scaled_encoded = pca_model_sklearn.transform(sets_test_scaled[0]) # test_data_scaled_decoded = pca_model_sklearn.inverse_transform(test_data_scaled_encoded) # our own model def pca_on_normalised(): params = {'latent_dim': 2} pca_model = PCAModel(params) pca_model.train(np.vstack(sets_training_scaled)) test_data_scaled_encoded = pca_model.encode(sets_test_scaled[0]) test_data_scaled_decoded = pca_model.decode(test_data_scaled_encoded) print("sets_test_scaled[0].shape", sets_test_scaled[0].shape) print("test_data_scaled_encoded.shape", test_data_scaled_encoded.shape) print("test_data_scaled_decoded.shape", test_data_scaled_decoded.shape) # plot results plotting.plot_2d(test_data_scaled_encoded, "wti_nymex_encoded_pca") simulated = preprocess_normalisation.rescale_data( test_data_scaled_decoded, dataset_name=test_dataset_names[0]) plotting.plot_some_curves("wti_nymex_normalised_compare_pca", sets_test[0], simulated, [25, 50, 75, 815], maturities) # plotting.plot_some_curves("test_feature_normalised_compare_normalisation", sets_test[0], sets_test_scaled[0], # [25, 50, 75, 815, 100, 600, 720, 740], maturities, plot_separate=True) # print("reconstruction_error", reconstruction_error(sets_test_scaled[0], test_data_scaled_decoded)) # print("reconstruction_error", reconstruction_error(np.array(sets_test[0]), simulated)) print("smape", smape(np.array(sets_test[0]), simulated)) # print("smape", np.mean(smape(np.array(sets_test[0]), simulated, over_curves=True))) def pca_on_unnormalised(): pca_model = PCAModel(k=2) pca_model.train(np.vstack(sets_training)) test_data_encoded = pca_model.encode(np.array(sets_test[0])) test_data_decoded = pca_model.decode(test_data_encoded) # plot results plotting.plot_2d(test_data_encoded.T, "wti_nymex_pca") # simulated = preprocess_normalisation.rescale_data(test_data_decoded, dataset_name=test_dataset_names[0]) plotting.plot_some_curves("wti_nymex_compare_pca", sets_test[0], test_data_decoded, [25, 50, 75, 815], maturities) # pca_on_unnormalised() pca_on_normalised()