def run_LSTM(X_train, X_valid, y_train, y_test, y_name, y_train_mean, y_train_std, save_folder): # ### 4H. LSTM (Long Short Term Memory) print 'head items to fit are: ', y_name # In[ ]: for head_item in range(len(y_name)): y_train_item = y_train[:, head_item] y_train_item = np.reshape(y_train_item, [y_train.shape[0], 1]) y_test_item = y_test[:, head_item] y_test_item = np.reshape(y_test_item, [y_test_item.shape[0], 1]) print '********************************** Fitting Deep Net on %s Data **********************************' % y_name[ head_item] #Declare model model_lstm = LSTMDecoder(dropout=0.25, num_epochs=5) model_lstm.get_means( y_train_mean, y_train_std) ### for un-zscoring during loss calculation ??? #Fit model model_lstm.fit(X_train, y_train_item) #Get predictions y_valid_predicted_lstm = model_lstm.predict(X_valid) training_prediction = model_lstm.predict(X_train) R2s_training = get_R2(y_train_item, training_prediction) print 'R2 on training set = ', R2s_training #Get metric of fit R2s_lstm = get_R2(y_test_item, y_valid_predicted_lstm) print('R2s:', R2s_lstm) print 'saving prediction ...' np.savez(save_folder + y_name[head_item] + '_LSTM_ypredicted.npz', y_test=y_test_item, y_prediction=y_valid_predicted_lstm, y_train_=y_train_item, training_prediction=training_prediction, y_train_mean=y_train_mean[head_item], y_train_std=y_train_std[head_item]) #print 'saving model ...' #joblib.dump(model_lstm, y_name[head_item] + '_LSTM.pkl') print 'plotting results...' plot_results(y_test_item, y_valid_predicted_lstm, y_name[head_item], R2s_lstm, model_name='LSTM', save_folder=save_folder) return model_lstm
def ridgeCV_model(X_train, X_valid, y_train, y_test, y_name, y_train_mean, y_train_std): print 'head items to fit are: ', y_name # In[ ]: for head_item in range(len(y_name)): y_train_item = y_train[:, head_item] y_train_item = np.reshape(y_train_item, [y_train.shape[0], 1]) y_test_item = y_test[:, head_item] y_test_item = np.reshape(y_test_item, [y_test_item.shape[0], 1]) print '********************************** Fitting RidgeCV on %s Data **********************************' % y_name[ head_item] #Declare model model = linear_model.RidgeCV(alphas=[0.1, 1.0, 10.0], normalize=True, fit_intercept=True) #Fit model model.fit(X_train, y_train_item) #Get predictions y_valid_predicted = model.predict(X_valid) training_prediction = model.predict(X_train) R2s_training = get_R2(y_train_item, training_prediction) print 'R2 on training set = ', R2s_training #Get metric of fit R2s = get_R2(y_test_item, y_valid_predicted) print('R2s:', R2s) print 'saving prediction ...' np.savez(y_name[head_item] + '_RidgeCV_ypredicted.npz', y_test=y_test_item, y_prediction=y_valid_predicted, y_train_=y_train_item, training_prediction=training_prediction, y_train_mean=y_train_mean[head_item], y_train_std=y_train_std[head_item]) #print 'saving model ...' joblib.dump(model, y_name[head_item] + '_Ridge.pkl') print 'plotting results...' plot_results(y_test_item, y_valid_predicted, y_name[head_item], R2s, model_name='RidgeCV') return model
def WienerCascade(X_train, X_valid, y_train, y_test, y_name, y_train_mean, y_train_std): # ### 4B. Wiener Cascade (Linear Nonlinear Model) print 'head items to fit are: ', y_name # In[ ]: for head_item in range(len(y_name)): y_train_item = y_train[:, head_item] y_train_item = np.reshape(y_train_item, [y_train.shape[0], 1]) y_test_item = y_test[:, head_item] y_test_item = np.reshape(y_test_item, [y_test_item.shape[0], 1]) print '********************************** Fitting WienerCascade on %s Data **********************************' % y_name[ head_item] #Declare model model = WienerCascadeDecoder(degree=3) #Fit model model.fit(X_train, y_train_item) #Get predictions y_valid_predicted = model.predict(X_valid) training_prediction = model.predict(X_train) R2s_training = get_R2(y_train_item, training_prediction) print 'R2 on training set = ', R2s_training #Get metric of fit R2s = get_R2(y_test_item, y_valid_predicted) print('R2s:', R2s) print 'saving prediction ...' np.savez(y_name[head_item] + '_WienerCascade_ypredicted.npz', y_test=y_test_item, y_prediction=y_valid_predicted, y_train_=y_train_item, training_prediction=training_prediction, y_train_mean=y_train_mean[head_item], y_train_std=y_train_std[head_item]) #print 'saving model ...' joblib.dump(model, y_name[head_item] + '_WienerCascade.pkl') print 'plotting results...' plot_results(y_test_item, y_valid_predicted, y_name[head_item], R2s, model_name='WienerCascade') return model
def BayesianRidge_model(X_train, X_valid, y_train, y_test, y_name, y_train_mean, y_train_std): model_name = 'BayesianRidge' print 'head items to fit are: ', y_name # In[ ]: for head_item in range(len(y_name)): y_train_item = y_train[:, head_item] #y_train_item = np.reshape(y_train_item,[y_train.shape[0],1]) y_test_item = y_test[:, head_item] #y_test_item = np.reshape(y_test_item,[y_test_item.shape[0],1]) print '********************************** Fitting %s on %s Data **********************************' % ( model_name, y_name[head_item]) #Declare model model = linear_model.BayesianRidge(compute_score=True) #Fit model model.fit(X_train, y_train_item) #Get predictions y_valid_predicted = model.predict(X_valid) training_prediction = model.predict(X_train) R2s_training = get_R2(y_train_item, training_prediction) print 'R2 on training set = ', R2s_training #Get metric of fit R2s = get_R2(y_test_item, y_valid_predicted) print('R2s:', R2s) print 'saving prediction ...' np.savez(y_name[head_item] + '_%s_ypredicted.npz' % model_name, y_test=y_test_item, y_prediction=y_valid_predicted, y_train_=y_train_item, training_prediction=training_prediction, y_train_mean=y_train_mean[head_item], y_train_std=y_train_std[head_item]) #print 'saving model ...' joblib.dump(model, y_name[head_item] + '_%s.pkl' % model_name) print 'plotting results...' plot_results(y_test_item, y_valid_predicted, y_name[head_item], R2s, model_name=model_name) return model
def LSTM_evaluate(units,dropout,num_epochs): units=int(units) num_epochs=int(num_epochs) model_LSTM=LSTMDecoder(units,dropout,num_epochs) model_LSTM.fit(X_train,y_train) y_valid_pred=model_LSTM.predict(X_valid) return np.mean(get_R2(y_valid,y_valid_pred))
def run_LSTM(X_train, X_valid, y_train, y_valid): # ### 4H. LSTM (Long Short Term Memory) # In[ ]: #Declare model model_lstm = LSTMDecoder(units=400, dropout=0, num_epochs=5) #Fit model model_lstm.fit(X_train, y_train) #Get predictions y_valid_predicted_lstm = model_lstm.predict(X_valid) #Get metric of fit R2s_lstm = get_R2(y_valid, y_valid_predicted_lstm) print('R2s:', R2s_lstm) print 'Results: ' print 'y_valid_predicted_lstm = ', y_valid_predicted_lstm print 'y_valid = ', y_valid np.savez('lstm_ypredicted.npz', y_valid_predicted_lstm=y_valid_predicted_lstm, y_valid=y_valid) plot_results(y_valid, y_valid_predicted_lstm) return model_lstm
def DNN(X_train,X_valid,y_train,y_test,y_name): # ### 4E. Dense Neural Network print('head items to fit are: ', y_name) # In[ ]: for head_item in range(len(y_name)): y_train_item = y_train[:,head_item] y_train_item = np.reshape(y_train_item,[y_train.shape[0],1]) y_test_item = y_test[:,head_item] y_test_item = np.reshape(y_test_item,[y_test_item.shape[0],1]) print('********************************** Fitting DNN on %s Data **********************************' % y_name[head_item]) #Declare model model_dnn=DenseNNDecoder(units=[128,64,32],num_epochs=15) #Fit model model_dnn.fit(X_train,y_train_item) #Get predictions y_valid_predicted=model_dnn.predict(X_valid) #Get metric of fit R2s=get_R2(y_test_item,y_valid_predicted) print('R2s:', R2s) #print 'saving prediction ...' np.savez(y_name[head_item] + '_DNN_ypredicted.npz',y_test=y_test_item,y_prediction=y_valid_predicted) #print 'saving model ...' #joblib.dump(model_dnn, y_name[head_item] + '_LSTM.pkl') print('plotting results...') plot_results(y_test_item,y_valid_predicted,y_name[head_item],R2s,model_name='DNN') return model_dnn
def RNN(X_train,y_train,X_valid,y_valid,y_name): model_name = 'RNN' # ### 4F. Simple RNN #print '############################# RUNNING RNN #############################' # In[ ]: #Declare model model_rnn=SimpleRNNDecoder(units=400,dropout=0,num_epochs=100) for head_item in range(len(y_name)): ### fit one at a time and save/plot the results #print '########### Fitting RNN on %s data ###########' % y_name[head_item] y_train_item = y_train[:,head_item] y_train_item = np.reshape(y_train_item,[y_train.shape[0],1]) #print 'shape of y_train_item = ', y_train_item.shape y_valid_item = y_valid[:,head_item] y_valid_item = np.reshape(y_valid_item,[y_valid_item.shape[0],1]) model_rnn.fit(X_train,y_train_item) #Get predictions y_valid_predicted_rnn=model_rnn.predict(X_valid) #Get metric of fit R2s_rnn=get_R2(y_valid_item,y_valid_predicted_rnn) print(y_name[head_item], 'R2:', R2s_rnn) np.savez(y_name[head_item] + '_rnn_ypredicted.npz',y_valid=y_valid_item,y_valid_predicted_rnn=y_valid_predicted_rnn) plot_results(y_valid_item,y_valid_predicted_rnn,y_name[head_item],R2s_rnn,model_name)
def SVR(X_flat_train, X_flat_valid, y_train, y_valid, y_name): # ### 4D. SVR (Support Vector Regression) # In[40]: #The SVR works much better when the y values are normalized, so we first z-score the y values #They have previously been zero-centered, so we will just divide by the stdev (of the training set) y_train_std = np.nanstd(y_train, axis=0) y_zscore_train = y_train / y_train_std #y_zscore_test=y_test/y_train_std y_zscore_valid = y_valid / y_train_std #Declare model model_svr = SVRDecoder(C=.1, max_iter=10000, gamma=1e-5) #Fit model for head_item in range(len(y_name)): ### fit one at a time and save/plot the results print '########### Fitting SVR on %s data ###########' % y_name[ head_item] y_zscore_train_item = y_zscore_train[:, head_item] y_zscore_train_item = np.reshape(y_zscore_train_item, [y_zscore_train.shape[0], 1]) print 'shape of y_zscore_train_item = ', y_zscore_train_item.shape y_zscore_valid_item = y_zscore_valid[:, head_item] y_zscore_valid_item = np.reshape(y_zscore_valid_item, [y_zscore_valid_item.shape[0], 1]) model_svr.fit(X_flat_train, y_zscore_train_item) #Get predictions y_zscore_valid_predicted_svr = model_svr.predict(X_flat_valid) #Get metric of fit R2s_svr = get_R2(y_zscore_valid_item, y_zscore_valid_predicted_svr) print(y_name[head_item], 'R2:', R2s_svr) np.savez(y_name[head_item] + '_svr_ypredicted.npz', y_zscore_valid=y_zscore_valid_item, y_zscore_valid_predicted_svr=y_zscore_valid_predicted_svr) plot_results(y_zscore_valid_item, y_zscore_valid_predicted_svr, y_name[head_item], R2s_svr)
def GRU(): # ### 4G. GRU (Gated Recurrent Unit) # In[ ]: #Declare model model_gru = GRUDecoder(units=400, dropout=0, num_epochs=5) #Fit model model_gru.fit(X_train, y_train) #Get predictions y_valid_predicted_gru = model_gru.predict(X_valid) #Get metric of fit R2s_gru = get_R2(y_valid, y_valid_predicted_gru) print('R2s:', R2s_gru)
def RNN(): # ### 4F. Simple RNN # In[ ]: #Declare model model_rnn = SimpleRNNDecoder(units=400, dropout=0, num_epochs=5) #Fit model model_rnn.fit(X_train, y_train) #Get predictions y_valid_predicted_rnn = model_rnn.predict(X_valid) #Get metric of fit R2s_rnn = get_R2(y_valid, y_valid_predicted_rnn) print('R2s:', R2s_rnn)
def DNN(): # ### 4E. Dense Neural Network # In[ ]: #Declare model model_dnn = DenseNNDecoder(units=400, dropout=0.25, num_epochs=10) #Fit model model_dnn.fit(X_flat_train, y_train) #Get predictions y_valid_predicted_dnn = model_dnn.predict(X_flat_valid) #Get metric of fit R2s_dnn = get_R2(y_valid, y_valid_predicted_dnn) print('R2s:', R2s_dnn)
def XGBoost(): # ### 4C. XGBoost (Extreme Gradient Boosting) # In[ ]: #Declare model model_xgb = XGBoostDecoder(max_depth=3, num_round=200, eta=0.3, gpu=-1) #Fit model model_xgb.fit(X_flat_train, y_train) #Get predictions y_valid_predicted_xgb = model_xgb.predict(X_flat_valid) #Get metric of fit R2s_xgb = get_R2(y_valid, y_valid_predicted_xgb) print('R2s:', R2s_xgb)
def WienerCascade(): # ### 4B. Wiener Cascade (Linear Nonlinear Model) # In[39]: #Declare model model_wc = WienerCascadeDecoder(degree=3) #Fit model model_wc.fit(X_flat_train, y_train) #Get predictions y_valid_predicted_wc = model_wc.predict(X_flat_valid) #Get metric of fit R2s_wc = get_R2(y_valid, y_valid_predicted_wc) print('R2s:', R2s_wc)
def Wiener(X_flat_train, X_flat_valid, y_train, y_valid): #Declare model model_wf = WienerFilterDecoder() #Fit model model_wf.fit(X_flat_train, y_train) #Get predictions y_valid_predicted_wf = model_wf.predict(X_flat_valid) #Get metric of fit R2s_wf = get_R2(y_valid, y_valid_predicted_wf) print('R2s:', R2s_wf) #plot_results(y_valid,y_valid_predicted_wf) return model_wf
def SVR(): # ### 4D. SVR (Support Vector Regression) # In[40]: #The SVR works much better when the y values are normalized, so we first z-score the y values #They have previously been zero-centered, so we will just divide by the stdev (of the training set) y_train_std = np.nanstd(y_train, axis=0) y_zscore_train = y_train / y_train_std y_zscore_test = y_test / y_train_std y_zscore_valid = y_valid / y_train_std #Declare model model_svr = SVRDecoder(C=5, max_iter=4000) #Fit model model_svr.fit(X_flat_train, y_zscore_train) #Get predictions y_zscore_valid_predicted_svr = model_svr.predict(X_flat_valid) #Get metric of fit R2s_svr = get_R2(y_zscore_valid, y_zscore_valid_predicted_svr) print('R2s:', R2s_svr)
def AllDecoders(X,y): train_R2=[] test_R2=[] valid_R2=[] y_train_pred=[] y_test_pred=[] y_valid_pred=[] BestParams={} # hyperparameter sets of each model models=['RNN','LSTM','GRU'] # decoders = [ # SimpleRNNDecoder(), # LSTMDecoder(), # GRUDecoder()] # params=[{'units':(50,100.99),'dropout':(0,0.2),'num_epochs':(2,5.99)}] params=[{'units':(50,600),'dropout':(0,0.6),'num_epochs':(2,31)}] initpoints=10 niter=10 k=10 #split training, testing datasets, and Z score input 'X' and 'y' # X_train_temp, X_test, y_train_temp, y_test = train_test_split(X, y, test_size=0.2) # X_train, X_valid, y_train, y_valid = train_test_split(X_train_temp, y_train_temp, test_size=0.2) training_range=[.2,1] valid_range=[0,.1] testing_range=[.1,.2] num_examples=X.shape[0] training_set=np.arange(np.int(np.round(training_range[0]*num_examples)),np.int(np.round(training_range[1]*num_examples))) valid_set=np.arange(np.int(np.round(valid_range[0]*num_examples)),np.int(np.round(valid_range[1]*num_examples))) testing_set=np.arange(np.int(np.round(testing_range[0]*num_examples)),np.int(np.round(testing_range[1]*num_examples))) #Get training data X_train=X[training_set,:,:] y_train=y[training_set,:] #Get testing data X_test=X[testing_set,:,:] y_test=y[testing_set,:] #Get validation data X_valid=X[valid_set,:,:] y_valid=y[valid_set,:] X_mean=np.nanmean(X,axis=0) X_std=np.nanstd(X,axis=0) X_train=np.nan_to_num((X_train-X_mean)/X_std) X_valid=np.nan_to_num((X_valid-X_mean)/X_std) X_test=np.nan_to_num((X_test-X_mean)/X_std) #Zero-center outputs y_mean=np.mean(y,axis=0) y_train=y_train-y_mean y_test=y_test-y_mean y_valid=y_valid-y_mean # # RNN DECODERS # def RNN_evaluate(units,dropout,num_epochs): # units=int(units) # num_epochs=int(num_epochs) # model_RNN=SimpleRNNDecoder(units,dropout,num_epochs) # model_RNN.fit(X_train,y_train) # y_valid_pred=model_RNN.predict(X_valid) # return np.mean(get_R2(y_valid,y_valid_pred)) # # TUNING DECODERS # RNN_BO=BayesianOptimization(RNN_evaluate,params[0],verbose=0) # RNN_BO.maximize(init_points=initpoints, n_iter=niter, kappa=k) # best_params=RNN_BO.max['params'] # best_params['units']=int(best_params['units']) # best_params['num_epochs']=int(best_params['num_epochs']) # # predict test data # model_RNN=SimpleRNNDecoder(best_params['units'],best_params['dropout'],best_params['num_epochs']) # model_RNN.fit(X_train,y_train) # y_valid_pred_temp=model_RNN.predict(X_valid) # y_test_pred_temp=model_RNN.predict(X_test) # y_valid_pred.append(y_valid_pred_temp) # y_test_pred.append(y_test_pred_temp) # valid_R2.append(np.mean(get_R2(y_valid,y_valid_pred_temp))) # test_R2.append(np.mean(get_R2(y_test,y_test_pred_temp))) # BestParams[models[0]]=best_params # LSTM DECODERS def LSTM_evaluate(units,dropout,num_epochs): units=int(units) num_epochs=int(num_epochs) model_LSTM=LSTMDecoder(units,dropout,num_epochs) model_LSTM.fit(X_train,y_train) y_valid_pred=model_LSTM.predict(X_valid) return np.mean(get_R2(y_valid,y_valid_pred)) # TUNING DECODER LSTM_BO=BayesianOptimization(LSTM_evaluate,params[0],verbose=1) LSTM_BO.maximize(init_points=initpoints, n_iter=niter, kappa=k) best_params=LSTM_BO.max['params'] best_params['units']=int(best_params['units']) best_params['num_epochs']=int(best_params['num_epochs']) # predict test data model_LSTM=LSTMDecoder(best_params['units'],best_params['dropout'],best_params['num_epochs']) model_LSTM.fit(X_train,y_train) y_train_pred_temp=model_LSTM.predict(X_train) y_valid_pred_temp=model_LSTM.predict(X_valid) y_test_pred_temp=model_LSTM.predict(X_test) y_train_pred.append(y_train_pred_temp) y_valid_pred.append(y_valid_pred_temp) y_test_pred.append(y_test_pred_temp) train_R2.append(np.mean(get_R2(y_train,y_train_pred_temp))) valid_R2.append(np.mean(get_R2(y_valid,y_valid_pred_temp))) test_R2.append(np.mean(get_R2(y_test,y_test_pred_temp))) BestParams[models[1]]=best_params # GRU DECODERS def GRU_evaluate(units,dropout,num_epochs): units=int(units) num_epochs=int(num_epochs) model_GRU=GRUDecoder(units,dropout,num_epochs) model_GRU.fit(X_train,y_train) y_valid_pred=model_GRU.predict(X_valid) return np.mean(get_R2(y_valid,y_valid_pred)) # TUNING DECODERS GRU_BO=BayesianOptimization(GRU_evaluate,params[0],verbose=0) GRU_BO.maximize(init_points=initpoints, n_iter=niter, kappa=k) best_params=GRU_BO.max['params'] best_params['units']=int(best_params['units']) best_params['num_epochs']=int(best_params['num_epochs']) # predict test data model_GRU=GRUDecoder(best_params['units'],best_params['dropout'],best_params['num_epochs']) model_GRU.fit(X_train,y_train) y_train_pred_temp=model_GRU.predict(X_train) y_valid_pred_temp=model_GRU.predict(X_valid) y_test_pred_temp=model_GRU.predict(X_test) y_train_pred.append(y_train_pred_temp) y_valid_pred.append(y_valid_pred_temp) y_test_pred.append(y_test_pred_temp) train_R2.append(np.mean(get_R2(y_train,y_train_pred_temp))) valid_R2.append(np.mean(get_R2(y_valid,y_valid_pred_temp))) test_R2.append(np.mean(get_R2(y_test,y_test_pred_temp))) BestParams[models[2]]=best_params return train_R2, valid_R2, test_R2, y_train_pred, y_valid_pred, y_test_pred, y_train, y_valid, y_test, BestParams
#Get training data X_train = X[train, :, :] y_train = y[train, :] #Get testing data X_test = X[test, :, :] y_test = y[test, :] #Fit model model_rnn.fit(X_train, y_train) #Get predictions y_test_predicted_rnn = model_rnn.predict(X_test) #Get metric of fit R2s_tmp[i, j, :] = get_R2(y_test, y_test_predicted_rnn) print('R2s:', R2s_tmp[i, j, :]) plt.errorbar(fractions_of_data**-1, np.mean(R2s_tmp[:, :, 1], axis=1), np.std(R2s_tmp[:, :, 1], axis=1)) plt.savefig("perf_over_data_dt" + str(60 * dt_ratio) + ".eps") ##Declare model #model_lstm=LSTMDecoder(units=400,dropout=0,num_epochs=5) # ##Fit model #model_lstm.fit(X_train,y_train) # ##Get predictions #y_valid_predicted_lstm = model_lstm.predict(X_valid)