def run_TESTS(): data = np.array(run(21100)).reshape(-1, 1) data_mean = np.mean(data) split = 20000 X_train = data[:split - 1] y_train = data[1:split] X_test = data[split - 1:-1] y_test = data[split:] lcesn = LCESN(input_size=1, output_size=1, num_reservoirs=3, reservoir_sizes=300, echo_params=0.85, init_echo_timesteps=100, regulariser=1e-6) lcesn.initialize_input_weights(strategies='binary', scales=1.) lcesn.initialize_reservoir_weights(strategies='uniform', spectral_scales=1.25) print('LCESN MADE') eesn = EESN(input_size=1, output_size=1, num_reservoirs=3, reservoir_sizes=300, echo_params=0.85, init_echo_timesteps=100, regulariser=1e-6) eesn.initialize_input_weights(strategies='binary', scales=1.) eesn.initialize_reservoir_weights(strategies='uniform', spectral_scales=1.25) print('EESN MADE') print('=' * 30) st_time = time.time() lcesn.train(X_train, y_train) print('LCESN TRAINED. TOOK %.3f SEC' % (time.time() - st_time)) st_time = time.time() eesn.train(X_train, y_train) print('EESN TRAINED. TOOK %.3f SEC' % (time.time() - st_time)) lcesn_outs = [] eesn_outs = [] for i, x in enumerate(X_test): lcesn_outs.append(lcesn.forward(x)) eesn_outs.append(eesn.forward(x)) lcesn_outs = np.array(lcesn_outs).squeeze() eesn_outs = np.array(eesn_outs).squeeze() fig, ax = plt.subplots() ax.plot(range(len(lcesn_outs)), lcesn_outs, label='lcesn') ax.plot(range(len(eesn_outs)), eesn_outs, label='new') ax.plot(range(len(y_test)), y_test, label='true') plt.show()
def load_and_run_model(file_name): model = pkl.load(open(file_name, "rb")) (seq, gen_train_loss, gen_test_loss, epoch) = model print("MODEL LOADED") from MackeyGlass.MackeyGlassGenerator import run data = run(21000) data -= np.mean(data) DATA_MEAN = np.mean(data) print("DATA LOADED") train_data = np.array(data[:14000]) train_inputs = Variable(torch.from_numpy(train_data.reshape(-1, 1)), requires_grad=0) test_data = np.array(data[14000:20000]) test_targets = Variable(torch.from_numpy(test_data.reshape(-1, 1)), requires_grad=0) val_data = np.array(data[20000:]) val_targets = Variable(torch.from_numpy(val_data.reshape(-1, 1)), requires_grad=0) pre_val_inputs = Variable(torch.from_numpy(np.array(data[:20000]).reshape(-1, 1)), requires_grad=0) gen_train_outs = seq.forward(train_inputs[:4000], future=2000).data.numpy() gen_train_loss = nrmse(gen_train_outs[4000:], train_inputs[4000:6000].data.numpy(), DATA_MEAN) print('!! Gen Train loss: {}'.format(gen_train_loss)) gen_test_outs = seq.forward(train_inputs, future=2000).data.numpy() gen_test_loss = nrmse(gen_test_outs[14000:], test_targets.data.numpy()[:2000], DATA_MEAN) print('!! Gen Test loss: {}'.format(gen_test_loss)) gen_val_outs = seq.forward(pre_val_inputs, future=1000).data.numpy() gen_val_loss = nrmse(gen_val_outs[20000:], val_targets.data.numpy()[:1000], DATA_MEAN) print('!! Gen Val loss: {}'.format(gen_val_loss)) plt.plot(range(len(data)), data, label="T") plt.plot(range(len(gen_val_outs)), gen_val_outs, label="G") plt.legend() plt.show()
import numpy as np import matplotlib.pyplot as plt from ESN.ESN import EESN, ESN from MackeyGlass.MackeyGlassGenerator import run from Helper.utils import nrmse import pickle as pkl import itertools import time def mse(y1, y2): return np.mean((y1 - y2)**2) if __name__ == '__main__': data = np.array([run(15100)]).reshape(-1, 1) data_mean = np.mean(data, axis=0) split = 14100 X_train = np.array(data[:split-1]) y_train = np.array(data[1:split]) X_valid = np.array(data[split-1:-1]) y_valid = np.array(data[split:]) data_mean = np.mean(data) #esn = ESN(1, 1, 1000, echo_param=0.85, regulariser=1e-6) #esn.initialize_input_weights(scale=1.0) #esn.initialize_reservoir_weights(spectral_scale=1.25) num_reservoirs = 10 echo_params_ = [ np.linspace(0.85, 0.5, num_reservoirs), np.linspace(0.85, 0.85, num_reservoirs), np.linspace(0.9, 0.9, num_reservoirs), np.linspace(0.9, 0.5, num_reservoirs), np.linspace(0.9, 0.75, num_reservoirs), np.linspace(0.75, 0.9, num_reservoirs),
t = str(time.time()).replace('.', 'p') eval_valid = True # whether or not to evaluate MSE loss on test set during training eval_gener = True # whether or not to generate future values, calculate that MSE loss eval_gen_loss = True save_fig = False save_results = False reg = 1e-3 # lambda for L2 regularization n_generate_timesteps = 2000 learn_rate = 0.009 n_epochs = 100 # ======================================================================================== # Get data =============================================================================== from MackeyGlass.MackeyGlassGenerator import run data = run(num_data_samples=21000) data_var = np.var(np.array(data)) __DATA_VAR__ = np.var(np.array(data)) __DATA_MEAN__ = np.mean(np.array(data)) print('data mean, variance: %.5f, %.5f' % (__DATA_MEAN__, __DATA_VAR__)) train_data = data[:14000] if eval_valid: valid_data = data[14000:20000] else: valid_data = None test_data = data[20000:] # Set up model, loss function, optimizer ================================================= model = FFNN(input_size=50, hidden_size=100, n_hidden_layers=2, activation=nn.Sigmoid) criterion = nn.MSELoss()
def try_toy_example(): from MackeyGlass.MackeyGlassGenerator import run data = run(20000) data -= np.mean(data) DATA_MEAN = np.mean(data) train_data = np.array(data[:14000]); test_data = np.array(data[14000:]) # CONSTRUCT TRAINING, TESTING DATA train_inputs = Variable(torch.from_numpy(train_data[:-1].reshape(-1, 1)), requires_grad=0) train_targets = Variable(torch.from_numpy(train_data[1:].reshape(-1, 1)), requires_grad=0) test_inputs = Variable(torch.from_numpy(test_data[:-1].reshape(-1, 1)), requires_grad=0) test_targets = Variable(torch.from_numpy(test_data[1:].reshape(-1, 1)), requires_grad=0) # print(train_inputs) seq = Sequence() seq.double() # ??? what does this do? criterion = nn.MSELoss() optimizer = optim.Adam(seq.parameters(), lr=0.01) bestGenTrain = 1000 bestGenTest = 1000 epoch = -1 bestModel = (None, bestGenTrain, bestGenTest, epoch) NUM_EPOCH = 500 for i in range(NUM_EPOCH): print('Epoch [{}/{}]'.format(i+1, NUM_EPOCH)) # calculate outputs, loss, then step optimizer.zero_grad() train_outputs = seq(train_inputs) loss = criterion(train_outputs, train_targets) print('Training loss: %.6f' % loss.data.cpu().numpy()[0]) loss.backward() optimizer.step() test_outputs = seq(test_inputs, future=0) loss = criterion(test_outputs, test_targets) print('Test loss: %.6f' % loss.data.cpu().numpy()[0]) if i % 5 == 0: gen_outs = seq.forward(train_inputs[:4000], future=2000).data.numpy() gen_train_loss = nrmse(gen_outs[4000:], train_targets.data.numpy()[4000:6000], DATA_MEAN) print('!! Gen Train loss: {}'.format(gen_train_loss)) gen_outs = seq.forward(train_targets[-4000:], future=2000).data.numpy() gen_test_loss = nrmse(gen_outs[4000:], test_inputs.data.numpy()[:2000], DATA_MEAN) print('!! Gen Test loss: {}'.format(gen_test_loss)) if gen_test_loss <= bestModel[2]: bestModel = (deepcopy(seq), gen_train_loss, gen_test_loss, i) pkl.dump(bestModel, open("bestModel_50hids.pkl", "wb")) print("---BEST SAVED") f, ax = plt.subplots(figsize=(12, 12)) # plot true test target values out_plt = test_outputs.data.cpu().numpy(); tar_plt = test_targets.data.cpu().numpy() ax.plot(np.arange(len(out_plt)), tar_plt, label='True') ax.plot(np.arange(len(out_plt)), out_plt, label='Generated') # generate data for final model f2, ax2 = plt.subplots(figsize=(12, 12)) outs = seq.forward(train_inputs[:100], future=2000).data.numpy() ax2.plot(np.arange(len(train_targets.data.numpy()[100:2100])), train_targets.data.numpy()[100:2100], label="True") ax2.plot(np.arange(len(outs[100:])), outs[100:2100], label="Predicted") ax2.set_title("GENERATIVE TRAIN DATA (FINAL MODEL)") print("FINAL GEN. LOSS FINAL MODEL --- TRAIN {}".format(nrmse(outs[100:], train_targets.data.numpy()[100:2100], DATA_MEAN))) f2, ax2 = plt.subplots(figsize=(12, 12)) outs = seq.forward(test_inputs[:100], future=2000).data.numpy() ax2.plot(np.arange(len(test_targets.data.numpy()[100:2100])), test_targets.data.numpy()[100:2100], label="True") ax2.plot(np.arange(len(outs[100:])), outs[100:2100], label="Predicted") ax2.set_title("GENERATIVE TEST DATA (FINAL MODEL)") print("FINAL GEN. LOSS FINAL MODEL --- TRAIN {}".format(nrmse(outs[100:], test_targets.data.numpy()[100:2100], DATA_MEAN))) # generate data for best model f2, ax2 = plt.subplots(figsize=(12, 12)) outs = bestModel[0].forward(train_inputs[:100], future=2000).data.numpy() ax2.plot(np.arange(len(train_targets.data.numpy()[100:2100])), train_targets.data.numpy()[100:2100], label="True") ax2.plot(np.arange(len(outs[100:])), outs[100:2100], label="Predicted") ax2.set_title("GENERATIVE TRAIN DATA (BEST MODEL)") print("FINAL GEN. LOSS BEST MODEL --- TRAIN {}".format(nrmse(outs[100:], train_targets.data.numpy()[100:2100], DATA_MEAN))) f2, ax2 = plt.subplots(figsize=(12, 12)) outs = bestModel[0].forward(test_inputs[:100], future=2000).data.numpy() ax2.plot(np.arange(len(test_targets.data.numpy()[100:2100])), test_targets.data.numpy()[100:2100], label="True") ax2.plot(np.arange(len(outs[100:])), outs[100:2100], label="Predicted") ax2.set_title("GENERATIVE TEST DATA (BEST MODEL)") print("FINAL GEN. LOSS BEST MODEL --- TEST {}".format(nrmse(outs[100:], test_targets.data.numpy()[100:2100], DATA_MEAN))) plt.legend(); plt.show()
import datetime # def mse(y1, y2): # return np.mean((y1-y2)**2) # def nrmse(y_true, y_pred, MEAN_OF_DATA): # return np.sqrt(np.sum((y_true - y_pred)**2)/np.sum((y_true - MEAN_OF_DATA)**2)) def save_data(file_name, data_csv, delimiter, fmt, header): np.savetxt(file_name, data_csv, delimiter=delimiter, fmt=fmt, header=header) if __name__ == '__main__': data = np. array([run(15100)]).reshape(-1, 1) # NOTE: REMOVE WHEN NOT DHESN _std = np.std(data) #data -= np.mean(data) # data /= _std MEAN_OF_DATA = np.mean(data) split = 14100 X_train = np.array(data[:split-1]) y_train = np.array(data[1:split]) X_valid = np.array(data[split-1:-1]) y_valid = np.array(data[split:]) # print(np.mean(X_train)) # print(np.mean(X_valid)) # eesn = ESN(1, 1, 1000, echo_param=0.76800719, regulariser=1e-5)
for r in range(num_rows): for c in range(num_cols): idx = r*num_cols + c s_data = signals_data[idx] ax[r,c].plot(s_data) if len(titles) > 0: ax[r, c].set_title(titles[idx]) #plt.show(block=False) plt.draw() if __name__ == "__main__": data = np.array([run(21100)]).T # data_train_test = data[:20000] # data_val = data[20000:] #data = np.loadtxt('../../../MackeyGlass_t17.txt') #data -= np.mean(data) #data += 100.0 #data -= np.mean(data) #print(np.std(data)) #data /= np.std(data) #data *= 2.0 # onExit(data) #esn = ESN(input_size=2, output_size=1, reservoir_size=1000, echo_param=0.1, spectral_scale=1.1, init_echo_timesteps=100, regulariser=1e-0, debug_mode=True) #ESN_stochastic_train(data, 7000, esn, 1) MEAN_OF_DATA = np.mean(data) print("DATA MEAN: {}".format(MEAN_OF_DATA)) #np.random.seed(42)
def run_ESN(): data = np.array(run(21100)).reshape(-1, 1) data_mean = np.mean(data) split = 20000 X_train = data[:split - 1] y_train = data[1:split] X_test = data[split - 1:-1] y_test = data[split:] OLD_ESN = ESN2(input_size=1, output_size=1, reservoir_size=1000, echo_param=0.85, spectral_scale=1.25, init_echo_timesteps=100, regulariser=1e-6, input_weights_scale=1.) print('OLD ESN MADE') NEW_ESN = ESN(input_size=1, output_size=1, reservoir_size=1000, echo_param=0.85, init_echo_timesteps=100, regulariser=1e-6) NEW_ESN.initialize_input_weights(strategy='binary', scale=1.) NEW_ESN.initialize_reservoir_weights(strategy='uniform', spectral_scale=1.25) print('NEW ESN MADE') print('=' * 30) # Set new ESN's weights = old ESN's weights to ensure they (SHOULD) output the same outputs NEW_ESN.reservoir.W_res = OLD_ESN.W_reservoir NEW_ESN.reservoir.W_in = OLD_ESN.W_in assert np.sum(abs(NEW_ESN.reservoir.W_res - OLD_ESN.W_reservoir)) < 1e-3 if 0: for x0 in X_train: x0 = x0.reshape(-1, 1) old_res_fwd, in_old, res_old, old_st_old = OLD_ESN.__forward_to_res__( x0, debug=True) new_res_fwd, in_new, res_new, old_st_new = NEW_ESN.reservoir.forward( x0, debug=True) print('init state diff', np.sum(old_res_fwd - new_res_fwd)) print(' in_to_res diff: ', np.sum(in_old - in_new)) print('res_to_res diff: ', np.sum(res_old - res_new)) print(' old state diff: ', np.sum(old_st_old - old_st_new)) raw_input() # TRAIN THE NETWORKS ================================================= st_time = time.time() OLD_ESN.train(X_train, y_train) # S, D print('OLD ESN TRAINED. TOOK %.3f SEC' % (time.time() - st_time)) st_time = time.time() NEW_ESN.train(X_train, y_train) print('NEW ESN TRAINED. TOOK %.3f SEC' % (time.time() - st_time)) #print('success?', (OLD_ESN.W_out == NEW_ESN.W_out)) # diffs = (NEW_ESN.W_out - OLD_ESN.W_out).flatten() #plt.plot(range(len(diffs)), diffs) #plt.show() old_outputs = [] new_outputs = [] for i, x in enumerate(X_test): old_outputs.append(OLD_ESN.forward_to_out(x)) new_outputs.append(NEW_ESN.forward(x)) old_outputs = np.array(old_outputs).squeeze() new_outputs = np.array(new_outputs).squeeze() #print(old_outputs.shape) #print(old_outputs.shape, old_outputs[0].shape, old_outputs[0]) # if 1: # f, axarr = plt.subplots(1, 1, sharex=True, sharey=True, figsize=(12, 12)) # axarr[0].plot(range(len(old_outputs)), old_outputs, label='old') # axarr[1].plot(range(len(new_outputs)), new_outputs, label='new') # axarr[2].plot(range(len(y_test)), y_test, label='true') # plt.legend(); plt.show() # f.close() fig, ax = plt.subplots() ax.plot(range(len(old_outputs)), old_outputs, label='old') ax.plot(range(len(new_outputs)), new_outputs, label='new') ax.plot(range(len(y_test)), y_test, label='true') plt.show()