def preprocess(jerk, neural_data): # ## 3. Preprocess Data # ### 3A. User Inputs # The user can define what time period to use spikes from (with respect to the output). # In[25]: bins_before = 150 #How many bins of neural data prior to the output are used for decoding bins_current = 1 #Whether to use concurrent time bin of neural data bins_after = 150 #How many bins of neural data after the output are used for decoding # ### 3B. Format Covariates # #### Format Input Covariates # In[26]: # Format for recurrent neural networks (SimpleRNN, GRU, LSTM) # Function to get the covariate matrix that includes spike history from previous bins X = get_spikes_with_history(neural_data, bins_before, bins_after, bins_current) # Format for Wiener Filter, Wiener Cascade, XGBoost, and Dense Neural Network #Put in "flat" format, so each "neuron / time" is a single feature X_flat = X.reshape(X.shape[0], (X.shape[1] * X.shape[2])) # #### Format Output Covariates # In[79]: #Set decoding output #y=jerk_power y = jerk # ### 3C. Split into training / testing / validation sets # Note that hyperparameters should be determined using a separate validation set. # Then, the goodness of fit should be be tested on a testing set (separate from the training and validation sets). # #### User Options # In[32]: #Set what part of data should be part of the training/testing/validation sets training_range = [0, 0.5] testing_range = [0.7, 0.85] valid_range = [0.5, 1] # #### Split Data # In[81]: num_examples = X.shape[0] #Note that each range has a buffer of"bins_before" bins at the beginning, and "bins_after" bins at the end #This makes it so that the different sets don't include overlapping neural data training_set = np.arange( np.int(np.round(training_range[0] * num_examples)) + bins_before, np.int(np.round(training_range[1] * num_examples)) - bins_after) testing_set = np.arange( np.int(np.round(testing_range[0] * num_examples)) + bins_before, np.int(np.round(testing_range[1] * num_examples)) - bins_after) valid_set = np.arange( np.int(np.round(valid_range[0] * num_examples)) + bins_before, np.int(np.round(valid_range[1] * num_examples)) - bins_after) #Get training data X_train = X[training_set, :, :] X_flat_train = X_flat[training_set, :] y_train = y[training_set, :] #Get testing data X_test = X[testing_set, :, :] X_flat_test = X_flat[testing_set, :] y_test = y[testing_set, :] #Get validation data X_valid = X[valid_set, :, :] X_flat_valid = X_flat[valid_set, :] y_valid = y[valid_set, :] # ### 3D. Process Covariates # We normalize (z_score) the inputs and zero-center the outputs. # Parameters for z-scoring (mean/std.) should be determined on the training set only, and then these z-scoring parameters are also used on the testing and validation sets. # In[ ]: # In[82]: #Z-score "X" inputs. X_train_mean = np.nanmean(X_train, axis=0) X_train_std = np.nanstd(X_train, axis=0) X_train = (X_train - X_train_mean) / X_train_std X_test = (X_test - X_train_mean) / X_train_std X_valid = (X_valid - X_train_mean) / X_train_std #Z-score "X_flat" inputs. X_flat_train_mean = np.nanmean(X_flat_train, axis=0) X_flat_train_std = np.nanstd(X_flat_train, axis=0) X_flat_train = (X_flat_train - X_flat_train_mean) / X_flat_train_std X_flat_test = (X_flat_test - X_flat_train_mean) / X_flat_train_std X_flat_valid = (X_flat_valid - X_flat_train_mean) / X_flat_train_std #Zero-center outputs y_train_mean = np.mean(y_train, axis=0) y_train = y_train - y_train_mean y_test = y_test - y_train_mean y_valid = y_valid - y_train_mean print 'X_flat_train.shape, y_train.shape = ', X_flat_train.shape, y_train.shape return X_flat_train, X_flat_valid, X_train, X_valid, y_train, y_valid
TestY = matData['SmoothedFinger'] TestY = TestY[:, Finger] TestY = TestY.reshape(TestY.shape[0], 1) # preprocessing TrainX = remove_outliers(TrainX) x_scaler, y_scaler, TrainX, TestX, TrainY, TestY = preprocessing( TrainX, TestX, TrainY, TestY) # from here, we reconstruct the input by "looking back" a few steps bins_before = 20 #How many bins of neural data prior to the output are used for decoding bins_current = 1 #Whether to use concurrent time bin of neural data bins_after = 0 #How many bins of neural data after the output are used for decoding TrainX = get_spikes_with_history(TrainX, bins_before, bins_after, bins_current) TrainX, TrainY = TrainX[bins_before:, :, :], TrainY[bins_before:, ] TestX = get_spikes_with_history(TestX, bins_before, bins_after, bins_current) TestX, TestY = TestX[bins_before:, :, :], TestY[bins_before:, ] # Now, we reconstructed TrainX/TestX to have a shape (num_of_samples, sequence_length, input_size) # We can fit this to the LSTM print("Run for subject " + str(Idx_subject) + " finger " + str(Finger)) n_hidden = 20 n_layers = 5 input_dim = TrainX.shape[2] output_dim = TrainY.shape[1]
from metrics import get_R2 from metrics import get_rho # load data folder='./InputData/Decoding_Data/' #ENTER THE FOLDER THAT YOUR DATA IS IN filename='example_data_m1' with open(folder+filename+'.pickle','rb') as f: neural_data,vels_binned=pickle.load(f,encoding='latin1') #If using python 3 bins_before=6 #How many bins of neural data prior to the output are used for decoding bins_current=1 #Whether to use concurrent time bin of neural data bins_after=6 #How many bins of neural data after the output are used for decoding # Format for recurrent neural networks (SimpleRNN, GRU, LSTM) # Function to get the covariate matrix that includes spike history from previous bins X=get_spikes_with_history(neural_data,bins_before,bins_after,bins_current) #Set decoding output y=vels_binned FireRate_trainR2, FireRate_validR2, FireRate_testR2, FireRate_y_train_pred, FireRate_y_valid_pred, FireRate_y_test_pred, FireRate_y_train, FireRate_y_valid, FireRate_y_test, FireRate_best_params=AllDecoders(X,y) with open(filename+'_training_results.pkl','wb') as f: pickle.dump([FireRate_trainR2, FireRate_validR2, FireRate_testR2, FireRate_y_train_pred, FireRate_y_valid_pred, FireRate_y_test_pred, FireRate_y_train, FireRate_y_valid, FireRate_y_test, FireRate_best_params],f) print('Best param using firerate: ') print(FireRate_best_params) Modcolors=['b','g'] Models=['LSTM','GRU']
folder = '' # folder='/home/jglaser/Data/DecData/' # folder='/Users/jig289/Dropbox/Public/Decoding_Data/' t0 = time() with open(folder + 'example_data_s1.pickle', 'rb') as f: # neural_data,vels_binned=pickle.load(f,encoding='latin1') #If using python 3 neural_data, vels_binned, dt_ratio = pickle.load(f) #If using python 2 print(dt_ratio) bins_before = 3 * dt_ratio #How many bins of neural data prior to the output are used for decoding bins_current = 1 #Whether to use concurrent time bin of neural data bins_after = 3 * dt_ratio #How many bins of neural data after the output are used for decoding # Format for recurrent neural networks (SimpleRNN, GRU, LSTM) # Function to get the covariate matrix that includes spike history from previous bins inputdata = get_spikes_with_history(neural_data, bins_before, bins_after, bins_current, dt_ratio) fractions_of_data = np.asarray([1, 0.8, 0.6, 0.4, 0.2, 0.05]) fractions_of_data = np.asarray([1, 0.8]) crossval = 2 #Z-score "X" inputs. inputdata_mean = np.nanmean(inputdata, axis=0) inputdata_std = np.nanstd(inputdata, axis=0) inputdata = (inputdata - inputdata_mean) / inputdata_std #Zero-center outputs vels_mean = np.mean(vels_binned, axis=0) vels_binned = vels_binned - vels_mean #Declare model