def test_designmat(tmpdir):
mat1 = Design_Matrix({'X':[1,4,2,7,5,9,2,1,3,2],'Y':[3,0,0,6,9,9,10,10,1,10],'Z':[2,2,2,2,7,0,1,3,3,2],'intercept':[1,1,1,1,1,1,1,1,1,1]},TR=2.0,hasIntercept=True)
mat2 = Design_Matrix({'X':[9,9,2,7,5,0,1,1,1,2],'Y':[3,3,3,6,9,0,1,10,1,10],'Z':[2,6,3,2,7,0,1,7,8,8],'intercept':[1,1,1,1,1,1,1,1,1,1]},TR=2.0,hasIntercept=True)
#appending
assert mat1.append(mat1,axis=1).shape == (mat1.shape[0],mat1.shape[1]+mat2.shape[1])
assert mat1.append(mat2,axis=0).shape == (mat1.shape[0]+mat2.shape[0],mat1.shape[1]+1)
#convolution doesn't affect intercept
assert all(mat1.convolve().iloc[:,-1] == mat1.iloc[:,-1])
#but it still works
assert (mat1.convolve().iloc[:,:3].values != mat1.iloc[:,:3].values).any()
#Test vifs
expectedVifs = np.array([ 1.03984251, 1.02889877, 1.02261945])
assert np.allclose(expectedVifs,mat1.vif())
#poly
mat1.addpoly(order=4).shape[1] == mat1.shape[1]+4
mat1.addpoly(order=4,include_lower=False).shape[1] == mat1.shape[1]+1
#zscore
z = mat1.zscore(colNames=['X','Z'])
assert (z['Y'] == mat1['Y']).all()
assert z.shape == mat1.shape
def test_append(sim_design_matrix):
mats = sim_design_matrix.append(sim_design_matrix)
assert mats.shape[0] == sim_design_matrix.shape[0] * 2
# Keep polys separate by default
assert (mats.shape[1] - 4) == (sim_design_matrix.shape[1] - 4) * 2
# Otherwise stack them
mats = sim_design_matrix.append(sim_design_matrix, keep_separate=False)
assert mats.shape[1] == sim_design_matrix.shape[1]
assert mats.shape[0] == sim_design_matrix.shape[0] * 2
# Keep a single stimulus column separate
assert (sim_design_matrix.append(sim_design_matrix,
unique_cols=["face_A"]).shape[1] == 5)
# Keep a common stimulus class separate
assert (sim_design_matrix.append(sim_design_matrix,
unique_cols=["face*"]).shape[1] == 6)
# Keep a common stimulus class and a different single stim separate
assert (sim_design_matrix.append(sim_design_matrix,
unique_cols=["face*",
"house_A"]).shape[1] == 7)
# Keep multiple stimulus class separate
assert (sim_design_matrix.append(sim_design_matrix,
unique_cols=["face*",
"house*"]).shape[1] == 8)
# Growing a multi-run design matrix; keeping things separate
num_runs = 4
all_runs = Design_Matrix(sampling_freq=0.5)
for i in range(num_runs):
run = Design_Matrix(
np.array([
[1, 0, 0, 0],
[1, 0, 0, 0],
[0, 0, 0, 0],
[0, 1, 0, 0],
[0, 1, 0, 0],
[0, 0, 0, 0],
[0, 0, 1, 0],
[0, 0, 1, 0],
[0, 0, 0, 0],
[0, 0, 0, 1],
[0, 0, 0, 1],
]),
sampling_freq=0.5,
columns=["stim_A", "stim_B", "cond_C", "cond_D"],
)
run = run.add_poly(2)
all_runs = all_runs.append(run, unique_cols=["stim*", "cond*"])
assert all_runs.shape == (44, 28)
def test_append(sim_design_matrix):
mats = sim_design_matrix.append(sim_design_matrix)
assert mats.shape[0] == sim_design_matrix.shape[0] * 2
# Keep polys separate by default
assert (mats.shape[1] - 4) == (sim_design_matrix.shape[1] - 4) * 2
# Otherwise stack them
assert sim_design_matrix.append(sim_design_matrix,
keep_separate=False).shape[1] == sim_design_matrix.shape[1]
# Keep a single stimulus column separate
assert sim_design_matrix.append(sim_design_matrix,
unique_cols=['face_A']).shape[1] == 5
# Keep a common stimulus class separate
assert sim_design_matrix.append(sim_design_matrix,
unique_cols=['face*']).shape[1] == 6
# Keep a common stimulus class and a different single stim separate
assert sim_design_matrix.append(sim_design_matrix,
unique_cols=['face*', 'house_A']).shape[1] == 7
# Keep multiple stimulus class separate
assert sim_design_matrix.append(sim_design_matrix,
unique_cols=['face*', 'house*']).shape[1] == 8
# Growing a multi-run design matrix; keeping things separate
num_runs = 4
all_runs = Design_Matrix(sampling_freq=.5)
for i in range(num_runs):
run = Design_Matrix(np.array([
[1, 0, 0, 0],
[1, 0, 0, 0],
[0, 0, 0, 0],
[0, 1, 0, 0],
[0, 1, 0, 0],
[0, 0, 0, 0],
[0, 0, 1, 0],
[0, 0, 1, 0],
[0, 0, 0, 0],
[0, 0, 0, 1],
[0, 0, 0, 1]
]),
sampling_freq=.5,
columns=['stim_A', 'stim_B', 'cond_C', 'cond_D']
)
run = run.add_poly(2)
all_runs = all_runs.append(run, unique_cols=['stim*', 'cond*'])
assert all_runs.shape == (44, 28)
def test_designmat(tmpdir):
mat1 = Design_Matrix({
'X':[1, 4, 2, 7, 5, 9, 2, 1, 3, 2],
'Y':[3, 0, 0, 6, 9, 9, 10, 10, 1, 10],
'Z':[2, 2, 2, 2, 7, 0, 1, 3, 3, 2],
'intercept':[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
},
sampling_rate=2.0,polys=['intercept'])
mat2 = Design_Matrix({
'X':[9, 9, 2, 7, 5, 0, 1, 1, 1, 2],
'Y':[3, 3, 3, 6, 9, 0, 1, 10, 1, 10],
'Z':[2, 6, 3, 2, 7, 0, 1, 7, 8, 8],
'intercept':[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
},
sampling_rate=2.0, polys=['intercept'])
# Appending
# Basic horz cat
new_mat = mat1.append(mat1,axis=1)
assert new_mat.shape == (mat1.shape[0], mat1.shape[1] + mat2.shape[1])
both_cols = list(mat1.columns) + list(mat1.columns)
assert all(new_mat.columns == both_cols)
# Basic vert cat
new_mat = mat1.append(mat1,axis=0)
assert new_mat.shape == (mat1.shape[0]*2, mat1.shape[1]+1)
# Advanced vert cat
new_mat = mat1.append(mat1,axis=0,keep_separate=False)
assert new_mat.shape == (mat1.shape[0]*2,mat1.shape[1])
# More advanced vert cat
new_mat = mat1.append(mat1,axis=0,add_poly=2)
assert new_mat.shape == (mat1.shape[0]*2, 9)
#convolution doesn't affect intercept
assert all(mat1.convolve().iloc[:, -1] == mat1.iloc[:, -1])
#but it still works
assert (mat1.convolve().iloc[:, :3].values != mat1.iloc[:, :3].values).any()
#Test vifs
expectedVifs = np.array([ 1.03984251, 1.02889877, 1.02261945])
assert np.allclose(expectedVifs,mat1.vif())
#poly
mat1.add_poly(order=4).shape[1] == mat1.shape[1]+4
mat1.add_poly(order=4, include_lower=False).shape[1] == mat1.shape[1]+1
#zscore
z = mat1.zscore(columns=['X', 'Z'])
assert (z['Y'] == mat1['Y']).all()
assert z.shape == mat1.shape
# clean
mat = Design_Matrix({
'X':[1, 4, 2, 7, 5, 9, 2, 1, 3, 2],
'A':[1, 4, 2, 7, 5, 9, 2, 1, 3, 2],
'Y':[3, 0, 0, 6, 9, 9, 10, 10, 1, 10],
'Z':[2, 2, 2, 2, 7, 0, 1, 3, 3, 2],
'C':[1, 4, 2, 7, 5, 9, 2, 1, 3, 2],
'intercept':[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
},
sampling_rate=2.0,polys=['intercept'])
mat = mat[['X','A','Y','Z','C','intercept']]
assert all(mat.clean().columns == ['X','Y','Z','intercept'])
# replace data
mat = Design_Matrix({
'X':[1, 4, 2, 7, 5, 9, 2, 1, 3, 2],
'A':[1, 4, 2, 7, 5, 9, 2, 1, 3, 2],
'Y':[3, 0, 0, 6, 9, 9, 10, 10, 1, 10],
'Z':[2, 2, 2, 2, 7, 0, 1, 3, 3, 2],
'C':[1, 4, 2, 7, 5, 9, 2, 1, 3, 2]
},
sampling_rate=2.0)
mat = mat.replace_data(np.ones((mat.shape[0],mat.shape[1]-1)),column_names=['a','b','c','d'])
assert(np.allclose(mat.values,1))
assert(all(mat.columns == ['a','b','c','d']))
#DCT basis_mat
mat = Design_Matrix(np.random.randint(2,size=(500,3)),sampling_rate=2.0)
mat = mat.add_dct_basis()
assert len(mat.polys) == 11
assert mat.shape[1] == 14
#Up and down sampling
mat = Design_Matrix(np.random.randint(2,size=(500,4)),sampling_rate=2.0,columns=['a','b','c','d'])
target = 1
assert mat.upsample(target).shape[0] == mat.shape[0]*2 - target*2
target = 4
assert mat.downsample(target).shape[0] == mat.shape[0]/2
# 2) Load in covariates for this run
covariatesFile = os.path.join(get_resource_path(),
'covariates_example.csv')
cov = pd.read_csv(covariatesFile)
cov = Design_Matrix(cov, sampling_freq=sampling_freq)
# 3) In the covariates, fill any NaNs with 0, add intercept and linear trends and dct basis functions
cov = cov.fillna(0)
# Retain a list of nuisance covariates (e.g. motion and spikes) which we'll also want to also keep separate for each run
cov_columns = cov.columns
cov = cov.add_poly(1).add_dct_basis()
# 4) Join the onsets and covariates together
full = dm.append(cov, axis=1)
# 5) Append it to the master Design Matrix keeping things separated by run
all_runs = all_runs.append(full, axis=0, unique_cols=cov.columns)
all_runs.heatmap(vmin=-1, vmax=1)
#########################################################################
# We can see the left most columns of our multi-run design matrix contain our conditions of interest (stacked across all runs), the middle columns includes separate run-wise nuisiance covariates (motion, spikes) and the right most columns contain run specific polynomials (intercept, trends, etc).
#########################################################################
# Data Diagnostics
# ----------------
#
# Let's actually check if our design is estimable. Design Matrix provides a few tools for cleaning up highly correlated columns (resulting in failure if trying to perform regression), replacing data, and computing collinearity. By default the `clean` method will drop any columns correlated at r >= .95
######################################################################### # Similar to adding polynomial terms, Design Matrix has multiple methods for data processing and transformation such as downsampling, upsampling, and z-scoring. Let's use the z-score method to normalize the covariates we just loaded. # Use pandas built-in fillna to fill NaNs in the covariates files introduced during the pre-processing pipeline, before z-scoring # Z-score takes an optional argument of which columns to z-score. Since we don't want to z-score any spikes, so let's select everything except that column cov = cov.fillna(0).zscore(cov.columns[:-1]) cov.heatmap(vmin=-1,vmax=1) ######################################################################### # Concatenate Multiple Design Matrices # ------------------------------------ # # A really nice feature of Design Matrix is simplified, but intelligent matrix concatentation. Here it's trivial to horizontally concatenate our convolved onsets and covariates, while keeping our column names and order. full = dm.append(cov,axis=1) full.heatmap(vmin=-1,vmax=1) ######################################################################### # But we can also intelligently vertically concatenate design matrices to handle say, different experimental runs, or participants. The method enables the user to indicate which columns to keep separated (if any) during concatenation or which to treat as extensions along the first dimension. By default the class will keep all polylnomial terms separated. This is extremely useful when building 1 large design matrix composed of several runs or participants with separate means. dm2 = dm.append(dm, axis=0) dm2.heatmap(vmin=-1,vmax=1) ######################################################################### # Specific columns of interest can also be kept separate during concatenation (e.g. keeping run-wise spikes separate). As an example, we treat our first experimental regressor as different across our two design matrices. Notice that the class also preserves (as best as possible) column ordering. dm2 = dm.append(dm, axis=0, unique_cols=['BillyRiggins']) dm2.heatmap(vmin=-1,vmax=1) #########################################################################
