def preprocess(jerk, neural_data):
# ## 3. Preprocess Data
# ### 3A. User Inputs
# The user can define what time period to use spikes from (with respect to the output).
# In[25]:
bins_before = 150 #How many bins of neural data prior to the output are used for decoding
bins_current = 1 #Whether to use concurrent time bin of neural data
bins_after = 150 #How many bins of neural data after the output are used for decoding
# ### 3B. Format Covariates
# #### Format Input Covariates
# In[26]:
# Format for recurrent neural networks (SimpleRNN, GRU, LSTM)
# Function to get the covariate matrix that includes spike history from previous bins
X = get_spikes_with_history(neural_data, bins_before, bins_after,
bins_current)
# Format for Wiener Filter, Wiener Cascade, XGBoost, and Dense Neural Network
#Put in "flat" format, so each "neuron / time" is a single feature
X_flat = X.reshape(X.shape[0], (X.shape[1] * X.shape[2]))
# #### Format Output Covariates
# In[79]:
#Set decoding output
#y=jerk_power
y = jerk
# ### 3C. Split into training / testing / validation sets
# Note that hyperparameters should be determined using a separate validation set.
# Then, the goodness of fit should be be tested on a testing set (separate from the training and validation sets).
# #### User Options
# In[32]:
#Set what part of data should be part of the training/testing/validation sets
training_range = [0, 0.5]
testing_range = [0.7, 0.85]
valid_range = [0.5, 1]
# #### Split Data
# In[81]:
num_examples = X.shape[0]
#Note that each range has a buffer of"bins_before" bins at the beginning, and "bins_after" bins at the end
#This makes it so that the different sets don't include overlapping neural data
training_set = np.arange(
np.int(np.round(training_range[0] * num_examples)) + bins_before,
np.int(np.round(training_range[1] * num_examples)) - bins_after)
testing_set = np.arange(
np.int(np.round(testing_range[0] * num_examples)) + bins_before,
np.int(np.round(testing_range[1] * num_examples)) - bins_after)
valid_set = np.arange(
np.int(np.round(valid_range[0] * num_examples)) + bins_before,
np.int(np.round(valid_range[1] * num_examples)) - bins_after)
#Get training data
X_train = X[training_set, :, :]
X_flat_train = X_flat[training_set, :]
y_train = y[training_set, :]
#Get testing data
X_test = X[testing_set, :, :]
X_flat_test = X_flat[testing_set, :]
y_test = y[testing_set, :]
#Get validation data
X_valid = X[valid_set, :, :]
X_flat_valid = X_flat[valid_set, :]
y_valid = y[valid_set, :]
# ### 3D. Process Covariates
# We normalize (z_score) the inputs and zero-center the outputs.
# Parameters for z-scoring (mean/std.) should be determined on the training set only, and then these z-scoring parameters are also used on the testing and validation sets.
# In[ ]:
# In[82]:
#Z-score "X" inputs.
X_train_mean = np.nanmean(X_train, axis=0)
X_train_std = np.nanstd(X_train, axis=0)
X_train = (X_train - X_train_mean) / X_train_std
X_test = (X_test - X_train_mean) / X_train_std
X_valid = (X_valid - X_train_mean) / X_train_std
#Z-score "X_flat" inputs.
X_flat_train_mean = np.nanmean(X_flat_train, axis=0)
X_flat_train_std = np.nanstd(X_flat_train, axis=0)
X_flat_train = (X_flat_train - X_flat_train_mean) / X_flat_train_std
X_flat_test = (X_flat_test - X_flat_train_mean) / X_flat_train_std
X_flat_valid = (X_flat_valid - X_flat_train_mean) / X_flat_train_std
#Zero-center outputs
y_train_mean = np.mean(y_train, axis=0)
y_train = y_train - y_train_mean
y_test = y_test - y_train_mean
y_valid = y_valid - y_train_mean
print 'X_flat_train.shape, y_train.shape = ', X_flat_train.shape, y_train.shape
return X_flat_train, X_flat_valid, X_train, X_valid, y_train, y_valid
TestY = matData['SmoothedFinger']
TestY = TestY[:, Finger]
TestY = TestY.reshape(TestY.shape[0], 1)
# preprocessing
TrainX = remove_outliers(TrainX)
x_scaler, y_scaler, TrainX, TestX, TrainY, TestY = preprocessing(
TrainX, TestX, TrainY, TestY)
# from here, we reconstruct the input by "looking back" a few steps
bins_before = 20 #How many bins of neural data prior to the output are used for decoding
bins_current = 1 #Whether to use concurrent time bin of neural data
bins_after = 0 #How many bins of neural data after the output are used for decoding
TrainX = get_spikes_with_history(TrainX, bins_before, bins_after,
bins_current)
TrainX, TrainY = TrainX[bins_before:, :, :], TrainY[bins_before:, ]
TestX = get_spikes_with_history(TestX, bins_before, bins_after,
bins_current)
TestX, TestY = TestX[bins_before:, :, :], TestY[bins_before:, ]
# Now, we reconstructed TrainX/TestX to have a shape (num_of_samples, sequence_length, input_size)
# We can fit this to the LSTM
print("Run for subject " + str(Idx_subject) + " finger " + str(Finger))
n_hidden = 20
n_layers = 5
input_dim = TrainX.shape[2]
output_dim = TrainY.shape[1]
from metrics import get_R2
from metrics import get_rho
# load data
folder='./InputData/Decoding_Data/' #ENTER THE FOLDER THAT YOUR DATA IS IN
filename='example_data_m1'
with open(folder+filename+'.pickle','rb') as f:
neural_data,vels_binned=pickle.load(f,encoding='latin1') #If using python 3
bins_before=6 #How many bins of neural data prior to the output are used for decoding
bins_current=1 #Whether to use concurrent time bin of neural data
bins_after=6 #How many bins of neural data after the output are used for decoding
# Format for recurrent neural networks (SimpleRNN, GRU, LSTM)
# Function to get the covariate matrix that includes spike history from previous bins
X=get_spikes_with_history(neural_data,bins_before,bins_after,bins_current)
#Set decoding output
y=vels_binned
FireRate_trainR2, FireRate_validR2, FireRate_testR2, FireRate_y_train_pred, FireRate_y_valid_pred, FireRate_y_test_pred, FireRate_y_train, FireRate_y_valid, FireRate_y_test, FireRate_best_params=AllDecoders(X,y)
with open(filename+'_training_results.pkl','wb') as f:
pickle.dump([FireRate_trainR2, FireRate_validR2, FireRate_testR2, FireRate_y_train_pred, FireRate_y_valid_pred, FireRate_y_test_pred, FireRate_y_train, FireRate_y_valid, FireRate_y_test, FireRate_best_params],f)
print('Best param using firerate: ')
print(FireRate_best_params)
Modcolors=['b','g']
Models=['LSTM','GRU']
folder = ''
# folder='/home/jglaser/Data/DecData/'
# folder='/Users/jig289/Dropbox/Public/Decoding_Data/'
t0 = time()
with open(folder + 'example_data_s1.pickle', 'rb') as f:
# neural_data,vels_binned=pickle.load(f,encoding='latin1') #If using python 3
neural_data, vels_binned, dt_ratio = pickle.load(f) #If using python 2
print(dt_ratio)
bins_before = 3 * dt_ratio #How many bins of neural data prior to the output are used for decoding
bins_current = 1 #Whether to use concurrent time bin of neural data
bins_after = 3 * dt_ratio #How many bins of neural data after the output are used for decoding
# Format for recurrent neural networks (SimpleRNN, GRU, LSTM)
# Function to get the covariate matrix that includes spike history from previous bins
inputdata = get_spikes_with_history(neural_data, bins_before, bins_after,
bins_current, dt_ratio)
fractions_of_data = np.asarray([1, 0.8, 0.6, 0.4, 0.2, 0.05])
fractions_of_data = np.asarray([1, 0.8])
crossval = 2
#Z-score "X" inputs.
inputdata_mean = np.nanmean(inputdata, axis=0)
inputdata_std = np.nanstd(inputdata, axis=0)
inputdata = (inputdata - inputdata_mean) / inputdata_std
#Zero-center outputs
vels_mean = np.mean(vels_binned, axis=0)
vels_binned = vels_binned - vels_mean
#Declare model