def dummy_test(infile, expfile): # load input test data ifile = open(infile, "br") idic = pickle.load(ifile) ifile.close() slm1 = SLM(Term(1), Term(1)) slm2 = SLM(Term(1), Term(2)) for key in idic.keys(): if "1" in key: setattr(slm1, key[4:], idic[key]) elif "2" in key: setattr(slm2, key[4:], idic[key]) # run f test outdic = f_test(slm1, slm2) # load expected outout data efile = open(expfile, "br") expdic = pickle.load(efile) efile.close() testout = [] for key in expdic.keys(): comp = np.allclose(getattr(outdic, key), expdic[key], rtol=1e-05, equal_nan=True) testout.append(comp) assert all(flag == True for (flag) in testout)
def generate_random_two_slms(I): slm1 = SLM(FixedEffect(1), FixedEffect(1)) slm2 = SLM(FixedEffect(1), FixedEffect(2)) for key in I.keys(): if "1" in key: setattr(slm1, key[4:], I[key]) elif "2" in key: setattr(slm2, key[4:], I[key]) return slm1, slm2
def test_volumetric_input(): mask_image = nib.load( tflow.get("MNI152Lin", resolution="02", desc="brain", suffix="mask")) n_voxels = (mask_image.get_fdata() != 0).sum() n_subjects = 3 data = np.random.rand(n_subjects, n_voxels) model = FixedEffect(1) contrast = np.ones(3) slm = SLM(model, contrast, surf=mask_image) slm.fit(data)
def dummy_test(infile, expfile): # load input test data ifile = open(infile, "br") idic = pickle.load(ifile) ifile.close() slm = SLM(FixedEffect(1), FixedEffect(1)) for key in idic.keys(): setattr(slm, key, idic[key]) resels_py, reselspvert_py, edg_py = compute_resels(slm) out = {} out["resels"] = resels_py out["reselspvert"] = reselspvert_py out["edg"] = edg_py # load expected outout data efile = open(expfile, "br") expdic = pickle.load(efile) efile.close() testout = [] for key in out.keys(): if out[key] is not None and expdic[key] is not None: comp = np.allclose(out[key], expdic[key], rtol=1e-05, equal_nan=True) testout.append(comp) assert all(flag == True for (flag) in testout)
def dummy_test(infile, expfile): # load input test data ifile = open(infile, "br") idic = pickle.load(ifile) ifile.close() slm = SLM(FixedEffect(1), FixedEffect(1)) for key in idic.keys(): setattr(slm, key, idic[key]) # run _t_test t_test(slm) # load expected outout data efile = open(expfile, "br") expdic = pickle.load(efile) efile.close() testout = [] for key in expdic.keys(): comp = np.allclose(getattr(slm, key), expdic[key], rtol=1e-05, equal_nan=True) testout.append(comp) assert all(flag == True for (flag) in testout)
def dummy_test(infile, expfile): # load input test data ifile = open(infile, "br") idic = pickle.load(ifile) ifile.close() slm = SLM(FixedEffect(1), FixedEffect(1)) slm.t = idic["t"] slm.tri = idic["tri"] slm.mask = idic["mask"] slm.df = idic["df"] slm.k = idic["k"] slm.resl = idic["resl"] thresh = idic["thresh"] reselspvert = idic["reselspvert"] edg = idic["edg"] # call python function P_peak, P_clus, P_clusid = peak_clus(slm, thresh, reselspvert, edg) # load expected outout data efile = open(expfile, "br") expdic = pickle.load(efile) efile.close() O_peak = expdic["peak"] O_clus = expdic["clus"] O_clusid = expdic["clusid"] testout = [] if isinstance(P_peak, dict): for key in P_peak.keys(): comp = np.allclose(P_peak[key], O_peak[key], rtol=1e-05, equal_nan=True) testout.append(comp) else: comp = np.allclose(P_peak, O_peak, rtol=1e-05, equal_nan=True) if isinstance(P_clus, dict): for key in P_clus.keys(): comp = np.allclose(P_clus[key], O_clus[key], rtol=1e-05, equal_nan=True) else: comp = np.allclose(P_clus, O_clus, rtol=1e-05, equal_nan=True) testout.append(comp) testout.append(np.allclose(P_clusid, O_clusid, rtol=1e-05, equal_nan=True)) assert all(flag == True for (flag) in testout)
def test_SLM(): """Tests the SLM model using a grid of parameters Raises ------ Exception First exception that occurs in computing the SLM. """ samples = 10 predictors = 3 grid = list(create_parameter_grid(samples, predictors)) Y = np.random.rand(samples, 10242, predictors) for i in range(len(grid)): # Skip exceptions that we know error. if grid[i]["surf"] is None: if grid[i]["correction"] is not None and "rft" in grid[i][ "correction"]: continue if grid[i]["Y_idx"] > 1 and grid[i]["two_tailed"] is False: continue try: slm = SLM( model=grid[i]["model"], contrast=grid[i]["contrast"], surf=grid[i]["surf"], mask=grid[i]["mask"], correction=grid[i]["correction"], two_tailed=grid[i]["two_tailed"], ) slm.fit(Y[:, :, 0:grid[i]["Y_idx"]]) except Exception as e: print("Error on run:", i) print("SLM failed with the following parameters:") print("Model: ", grid[i]["model"]) print("Contrast: ", grid[i]["contrast"]) print("Surface: ", grid[i]["surf"]) print("Mask: ", grid[i]["mask"]) print("Correction: ", grid[i]["correction"]) print("Two_tailed: ", grid[i]["two_tailed"]) print("Y_idx: ", grid[i]["Y_idx"]) raise e
def dummy_test(infile, expfile): ifile = open(infile, "br") Din = pickle.load(ifile) ifile.close() Y = Din["Y"] M = Din["M"] # Convert M to a true BrainStat model fixed_effects = FixedEffect(M[:, Din["n_random"]:]) if Din["n_random"] != 0: mixed_effects = MixedEffect( M[:, :Din["n_random"]], name_ran=["f" + str(x) for x in range(Din["n_random"])], ) M = fixed_effects + mixed_effects else: M = fixed_effects # assign slm params slm = SLM(M, FixedEffect(1), surf=Din["surf"]) # here we go --> run the linear model slm._linear_model(Y) ofile = open(expfile, "br") Dout = pickle.load(ofile) ofile.close() # compare... testout = [] for k, v in Dout.items(): if k == "surf": # Surface data is only stored for reconstruction in MATLAB. continue a = getattr(slm, k) comp = np.allclose(a, v, rtol=1e-05, equal_nan=True) testout.append(comp) assert all(testout)
def generate_test_data(): np.random.seed(0) surface = _generate_sphere() parameters = [ { "n_observations": [103], "n_vertices": [np.array(get_points(surface)).shape[0]], "n_variates": [1, 2, 3], "n_predictors": [1, 7], "n_random": [0], "surf": [None, surface], }, { "n_observations": [103], "n_vertices": [np.array(get_points(surface)).shape[0]], "n_variates": [1], "n_predictors": [2, 7], "n_random": [1], "surf": [None, surface], }, ] test_num = 0 for params in ParameterGrid(parameters): test_num += 1 Y, M = generate_random_data_model( params["n_observations"], params["n_vertices"], params["n_variates"], params["n_predictors"], ) save_input_dict( {"Y": Y, "M": M, "surf": params["surf"], "n_random": params["n_random"]}, "xlinmod", test_num, ) model = array2effect(M, params["n_random"]) slm = SLM(model, FixedEffect(1), params["surf"]) slm.linear_model(Y) slm2files(slm, "xlinmod", test_num)
def dummy_test(infile, expfile): # load input test data ifile = open(infile, "br") idic = pickle.load(ifile) ifile.close() slm = SLM(Term(1), Term(1)) for key in idic.keys(): if key == "clusthresh": slm.cluster_threshold = idic[key] else: setattr(slm, key, idic[key]) empirical_output = random_field_theory(slm) # load expected outout data efile = open(expfile, "br") expdic = pickle.load(efile) efile.close() expected_output = (expdic["pval"], expdic["peak"], expdic["clus"], expdic["clusid"]) testout = [] for (empirical, expected) in zip(empirical_output, expected_output): if isinstance(expected, dict): for key in expected: if key == "mask": continue comp = np.allclose(empirical[key], expected[key], rtol=1e-05, equal_nan=True) testout.append(comp) else: if len(expected) is not 0: comp = np.allclose(empirical, expected, rtol=1e-05, equal_nan=True) testout.append(comp) assert all(flag == True for (flag) in testout)
def generate_t_test_out(I): slm = SLM(FixedEffect(1), FixedEffect(1)) for key in I.keys(): setattr(slm, key, I[key]) # run t_test t_test(slm) expkeys = ["X", "df", "coef", "SSE", "c", "k", "ef", "sd", "t"] D = {} for key in expkeys: D[key] = getattr(slm, key) return D
def dummy_test(infile, expfile, simple=True): ifile = open(infile, "br") Din = pickle.load(ifile) ifile.close() Y = Din["Y"] M = Din["M"] # assign slm params slm = SLM(M, FixedEffect(1)) if "tri" in Din: slm.surf = {"tri": Din["tri"]} if "lat" in Din: slm.surf = {"lat": Din["lat"]} # here we go --> run the linear model slm.linear_model(Y) ofile = open(expfile, "br") Dout = pickle.load(ofile) ofile.close() # compare... testout = [] for makey_ in Dout.keys(): comp = np.allclose(getattr(slm, makey_), Dout[makey_], rtol=1e-05, equal_nan=True) testout.append(comp) assert all(flag == True for (flag) in testout)
def get_linmod_output(Y, M, foutname, tri=None, lat=None): """Runs linmod and returns all relevant output.""" slm = SLM(M, FixedEffect(1)) if tri is not None: slm.surf = {"tri": tri} if lat is not None: slm.lat = {"lat": lat} slm.linear_model(Y) keys = [ "cluster_threshold", "coef", "df", "drlim", "niter", "resl", "SSE", "thetalim", "X", "tri", ] D = {} for key in keys: if getattr(slm, key) is not None: D[key] = getattr(slm, key) with open(foutname, "wb") as handle: pickle.dump(D, handle, protocol=4) return D
def generate_slm(**kwargs): """Generates a SLM with the given attributes Parameters ---------- All attributes of SLM can be provided as keyword arguments. Returns ------- brainstat.stats.SLM.SLM SLM object. """ slm = SLM(FixedEffect(1), 1) for key, value in kwargs.items(): setattr(slm, key, value) return slm
def copy_slm(slm): """Copies an SLM object. Parameters ---------- slm : brainstat.stats.SLM.SLM SLM object. Returns ------- brainstat.stats.SLM.SLM SLM object. """ slm_out = SLM(FixedEffect(1), 1) for key in slm.__dict__: setattr(slm_out, key, getattr(slm, key)) return slm_out
def get_fdr_output(D, foutname): """Runs fdr and returns all relevant output.""" slm = SLM(FixedEffect(1), FixedEffect(1)) for key in D.keys(): setattr(slm, key, D[key]) # run fdr Q = fdr(slm) Q_out = {} Q_out["Q"] = Q with open(foutname, "wb") as handle: pickle.dump(Q_out, handle, protocol=4) # return
def dummy_test(infile, expfile): # load input test data ifile = open(infile, "br") idic = pickle.load(ifile) ifile.close() slm = SLM(FixedEffect(1), FixedEffect(1)) for key in idic.keys(): setattr(slm, key, idic[key]) # run fdr Q = fdr(slm) # load expected outout data efile = open(expfile, "br") expdic = pickle.load(efile) efile.close() assert np.allclose(Q, expdic["Q"])
def dummy_test(infile, expfile): # load input test data ifile = open(infile, "br") idic = pickle.load(ifile) ifile.close() slm = SLM(Term(1), Term(1)) for key in idic.keys(): setattr(slm, key, idic[key]) # run fdr Q = fdr(slm) # load expected outout data # Note: expected dicts contain a "mask" key. this has been removed in our # current implementation. efile = open(expfile, "br") expdic = pickle.load(efile) efile.close() assert np.allclose(Q, expdic["Q"])
def generate_random_slm(rand_dict): """Generates a valid SLM for a surface. Parameters ---------- surf : BSPolyData or a dictionary with key 'tri' Brain surface. Returns ------- brainstat.stats.SLM SLM object. """ # this is going to be the input slm I = {} rand_slm = SLM(FixedEffect(1), FixedEffect(1)) for key in rand_dict.keys(): setattr(rand_slm, key, rand_dict[key]) I[key] = rand_dict[key] # this is going to be the output dict O = {} O["resels"], O["reselspvert"], O["edg"] = compute_resels(rand_slm) return I, O
def dummy_test(infile, expfile): # load input test data ifile = open(infile, "br") idic = pickle.load(ifile) ifile.close() model = array2effect(idic["M"], idic["n_random"]) contrast = -idic["M"][:, -1] # run _t_test slm = SLM(model, contrast, idic["surf"]) slm._linear_model(idic["Y"]) slm._t_test() # load expected outout data efile = open(expfile, "br") expdic = pickle.load(efile) efile.close() testout = [] for key in expdic.keys(): if isinstance(expdic[key], dict): slm_sub_dict = getattr(slm, key) exp_sub_dict = expdic[key] comp = np.all([ np.allclose(slm_sub_dict[x], exp_sub_dict[x]) for x in exp_sub_dict.keys() ]) else: comp = np.allclose(getattr(slm, key), expdic[key], rtol=1e-05, equal_nan=True) testout.append(comp) assert all(flag == True for (flag) in testout)
from brainstat.tutorial.utils import fetch_mics_data thickness, demographics = fetch_mics_data() mask = fetch_mask("fsaverage5") term_age = FixedEffect(demographics.AGE_AT_SCAN) term_sex = FixedEffect(demographics.SEX) term_subject = MixedEffect(demographics.SUB_ID) model = term_age + term_sex + term_age * term_sex + term_subject contrast_age = -model.mean.AGE_AT_SCAN slm = SLM( model, contrast_age, surf="fsaverage5", mask=mask, correction=["fdr", "rft"], two_tailed=False, cluster_threshold=0.01, ) slm.fit(thickness) ################################################################### # Genetics # -------- # # For genetic decoding we use the Allen Human Brain Atlas through the abagen # toolbox. Note that abagen only accepts parcellated data. Here is a minimal # example of how we use abagen to get the genetic expression of the 100 regions # of the Schaefer atlas and how to plot this expression to a matrix. Please note # that downloading the dataset and running this analysis can take several
term_iq = Term(iq, "iq") model = term_intercept + term_age + term_iq ################################################################### # We can also add interaction effects to the model by multiplying terms. model_interaction = term_intercept + term_age + term_iq + term_age * term_iq ################################################################### # Now, lets imagine we have some cortical marker (e.g. cortical thickness) for # each subject and we want to evaluate whether this marker changes with age # whilst correcting for effects of sex and age-sex interactions. from brainstat.stats.SLM import SLM slm = SLM(model_interaction, -age, surf=pial_left, correction="rft", mask=mask) slm.fit(thickness) print(slm.t.shape) # These are the t-values of the model. print(slm.P["pval"]["P"]) # These are the random field theory derived p-values. ################################################################### # By default BrainStat uses a two-tailed test. If you want to get a one-tailed # test, simply specify it in the SLM model as follows: slm_two_tails = SLM( model_interaction, -age, surf=pial_left, correction="rft", two_tailed=False ) slm_two_tails.fit(thickness) ################################################################### # Now, imagine that instead of using a fixed effects model, you would prefer a
print(model) ################################################################### # Now, imagine we have some cortical marker (e.g. cortical thickness) for each # subject, and we want to evaluate whether this marker is different across the # the lifespan. To do this, we can use the model we defined before, and a # contrast in observations (here: age). Then we simply initialize an SLM model # and fit it to the cortical thickness data. from brainstat.stats.SLM import SLM contrast_age = demographics.AGE_AT_SCAN slm_age = SLM( model, contrast_age, surf="fsaverage5", mask=mask, correction=["fdr", "rft"], cluster_threshold=0.01, ) slm_age.fit(thickness) ################################################################### # The resulting model, slm_age, will contain the t-statistic map, p-values # derived with the requested corrections, and a myriad of other properties (see # the API for more details). Lets plot the t-values and p-values on the surface. # We'll do this a few times throughout the tutorial so lets define a function to # do this. def plot_slm_results(slm, plot_peak=False, plot_fdr=False):
def dummy_test(infile, expfile): # load input test data ifile = open(infile, "br") idic = pickle.load(ifile) ifile.close() slm = SLM(FixedEffect(1), FixedEffect(1)) # Data are saved a little differently from the actual input due to compatibility with MATLAB. # Data wrangle a bit to bring it back into the Python input format. for key in idic.keys(): if key == "Y": # Y is input for slm.fit(), not a property. continue if key == "model": # Model is saved as a matrix rather than a Fixed/MixedEffect if idic[key].shape[1] == 1: idic[key] = FixedEffect(1) + FixedEffect(idic[key]) else: idic[key] = (FixedEffect(1) + FixedEffect(idic[key][:, 0]) + MixedEffect(idic[key][:, 1]) + MixedEffect(1)) setattr(slm, key, idic[key]) if key == "surf" and slm.surf is not None: slm.surf["tri"] += 1 slm.fit(idic["Y"]) # load expected outout data efile = open(expfile, "br") out = pickle.load(efile) efile.close() # Format of self.P changed since files were created -- alter out to match some changes. # Combine the list outputs, sort with pandas, and return to list. if "P" in out: out["P"]["pval"]["C"] = _onetailed_to_twotailed( out["P"]["pval"]["C"][0], out["P"]["pval"]["C"][1]) for key1 in ["peak", "clus"]: P_tmp = [] none_squeeze = lambda x: np.squeeze(x) if x is not None else None for i in range(len(out["P"][key1]["P"])): tail_dict = { key: none_squeeze(value[i]) for key, value in out["P"][key1].items() } if tail_dict["P"] is not None: if tail_dict["P"].size == 1: P_tmp.append(pd.DataFrame.from_dict([tail_dict])) else: P_tmp.append(pd.DataFrame.from_dict(tail_dict)) P_tmp[i].sort_values(by="P", ascending=True) else: P_tmp.append(pd.DataFrame(columns=tail_dict.keys())) out["P"][key1] = P_tmp testout = [] skip_keys = ["model", "correction", "_tri", "surf"] for key in out.keys(): if key in skip_keys: continue if key == "P": testout.append(recursive_comparison(out[key], getattr(slm, key))) elif out[key] is not None: comp = np.allclose(out[key], getattr(slm, key), rtol=1e-05, equal_nan=True) testout.append(comp) assert all(flag == True for (flag) in testout)
def generate_data_test_fdr(): ### test_01 data in-out generation print("test_fdr: test_01 data is generated..") # random data shape matching a real-data set # ['t'] : np array, shape (1, 64984), float64 # ['df'] : int # ['k'] : int t_dim = (1, 64984) df_max = 64984 finname = datadir("xstatq_01_IN.pkl") D = generate_random_fdr_data(t_dim, df_max, finname, seed=444) foutname = datadir("xstatq_01_OUT.pkl") get_fdr_output(D, foutname) ### test_02 data in-out generation print("test_fdr: test_02 data is generated..") # random data # ['t'] : np array, shape (1, 9850), float64 # ['df'] : int # ['k'] : int t_dim = (1, 9850) df_max = 1000 finname = datadir("xstatq_02_IN.pkl") D = generate_random_fdr_data(t_dim, df_max, finname, seed=445) foutname = datadir("xstatq_02_OUT.pkl") get_fdr_output(D, foutname) ### test_03 data in-out generation print("test_fdr: test_03 data is generated..") # similar to test_02, shapes/values of slm['t'] and slm['df'] manipulated # ['t'] : np array, shape (1, 2139), float64 # ['df'] : int # ['k'] : int t_dim = (1, 2139) df_max = 2000 k = 3 finname = datadir("xstatq_03_IN.pkl") D = generate_random_fdr_data(t_dim, df_max, finname, k=k, seed=446) foutname = datadir("xstatq_03_OUT.pkl") get_fdr_output(D, foutname) ### test_04 data in-out generation print("test_fdr: test_04 data is generated..") # similar to test_02 + optional input ['mask'] # ['t'] : np array, shape (1, 2475), float64 # ['df'] : int # ['k'] : int # ['mask'] : np array, shape (2475,), bool t_dim = (1, 2475) df_max = 1500 finname = datadir("xstatq_04_IN.pkl") mask_dim = 2475 D = generate_random_fdr_data(t_dim, df_max, finname, mask_dim=mask_dim, seed=447) foutname = datadir("xstatq_04_OUT.pkl") get_fdr_output(D, foutname) ### test_05 data in-out generation print("test_fdr: test_05 data is generated..") # similar to test_02 + optional input slm['dfs'] # ['t'] : np array, shape (1, 1998), float64 # ['df'] : int # ['k'] : int # ['dfs'] : np array, shape (1, 1998), int64 t_dim = (1, 1998) df_max = 4000 dfs_max = 1997 finname = datadir("xstatq_05_IN.pkl") D = generate_random_fdr_data(t_dim, df_max, finname, dfs_max=dfs_max, seed=448) foutname = datadir("xstatq_05_OUT.pkl") get_fdr_output(D, foutname) ### test_06 data in-out generation print("test_fdr: test_06 data is generated..") # similar to test_02 + optional inputs slm['dfs'] and ['mask'] # ['t'] : np array, shape (1, 3328), float64 # ['df'] : np array, shape (1, 1), int64 # ['k'] : int # ['dfs'] : np array, shape (1, 3328), int64 # ['mask'] : np array, shape (3328,), bool t_dim = (1, 3328) df_max = 10000 k = 2 dfs_max = 3328 mask_dim = 3328 finname = datadir("xstatq_06_IN.pkl") D = generate_random_fdr_data(t_dim, df_max, finname, k=k, dfs_max=dfs_max, mask_dim=mask_dim, seed=449) foutname = datadir("xstatq_06_OUT.pkl") get_fdr_output(D, foutname) ### test_07 data in-out generation print("test_fdr: test_07 data is generated..") # similar to test_02 + optional inputs slm['dfs'], ['mask'] and ['tri'] # ['t'] : np array, shape (1, 9512), float64 # ['df'] : int # ['k'] : int # ['dfs'] : np array, shape (1, 9512), int64 # ['mask'] : np array, shape (9512,), bool # ['tri'] : np array, shape (1724, 3), int64 t_dim = (1, 9512) df_max = 5000 dfs_max = 9511 mask_dim = 9512 tri_dim = (1724, 3) finname = datadir("xstatq_07_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, dfs_max=dfs_max, mask_dim=mask_dim, tri_dim=tri_dim, seed=450, ) foutname = datadir("xstatq_07_OUT.pkl") get_fdr_output(D, foutname) ### test_08 data in-out generation print("test_fdr: test_08 data is generated..") # similar to test_02 + optional inputs slm['dfs'], slm['tri'] and slm['resl'] # ['t'] : np array, shape (1, 1520), float64 # ['df'] : int # ['k'] : int # ['dfs'] : np array, shape (1, 1520), int64 # ['tri'] : np array, shape (4948, 3), int64 # ['resl'] : np array, shape (1520, 1), float64 t_dim = (1, 1520) df_max = 5000 k = 5 dfs_max = 9 tri_dim = (4948, 3) resl_dim = (1520, 1) finname = datadir("xstatq_08_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, k=k, dfs_max=dfs_max, tri_dim=tri_dim, resl_dim=resl_dim, seed=451, ) foutname = datadir("xstatq_08_OUT.pkl") get_fdr_output(D, foutname) ### test_09 data in-out generation print("test_fdr: test_09 data is generated..") # similar to test_08 + values/shapes of input params changed + # additional input slm['du'] (non-sense for _fdr) # ['t'] : np array, shape (1, 4397), float64 # ['df'] : int # ['k'] : int # ['tri'] : np array, shape (2734, 3), int64 # ['resl'] : np array, shape (8199, 1), float64 # ['dfs'] : np array, shape (1, 4397), float64 # ['du'] : int t_dim = (1, 14397) df_max = 1 dfs_max = 2 tri_dim = (2734, 3) resl_dim = (8199, 1) # du = 9 finname = datadir("xstatq_09_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, dfs_max=dfs_max, tri_dim=tri_dim, resl_dim=resl_dim, # du = du, seed=452, ) foutname = datadir("xstatq_09_OUT.pkl") get_fdr_output(D, foutname) ### test_10 data in-out generation print("test_fdr: test_10 data is generated..") # similar to test_08 + + values/shapes of input params changed + additional # input slm['du'], slm['c'], slm['ef'], and slm['sd'] (non-sense for _fdr) # ['t'] : np array, shape (1, 20484), float64 # ['df'] : int # ['k'] : int # ['tri'] : np array, shape (40960, 3), int32 # ['resl'] : np array, shape (61440, 1), float64 # ['c'] : np array, shape (1, 2), float64 # ['ef'] : np array, shape (1, 20484), float64 # ['sd'] : np array, shape (1, 20484), float64 t_dim = (1, 20484) df_max = 10 tri_dim = (40960, 3) resl_dim = (61440, 1) c_dim = (1, 2) ef_dim = (1, 20484) sd_dim = (1, 20484) finname = datadir("xstatq_10_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, tri_dim=tri_dim, resl_dim=resl_dim, c_dim=c_dim, ef_dim=ef_dim, sd_dim=sd_dim, seed=453, ) foutname = datadir("xstatq_10_OUT.pkl") get_fdr_output(D, foutname) ### test_11 data in-out generation print("test_fdr: test_11 data is generated..") # similar to test_08 + additional input ['c'], ['ef'], ['sd'], ['X'], # and ['coef'], ['SSE'] (non-sense for _fdr) # ['t'] : np array, shape (1, 20484), float64 # ['df'] : int # ['k'] : int # ['tri'] : np array, shape (40960, 3), int32 # ['resl'] : np array, shape (61440, 1), float64 # ['c'] : np array, shape (1, 2), float64 # ['ef'] : np array, shape (1, 20484), float64 # ['sd'] : np array, shape (1, 20484), float64 # ['X'] : np array, shape (10, 2), float64 # ['coef'] : np array, shape (2, 20484), float64 # ['SSE'] : np array, shape (1, 20484), float64 t_dim = (1, 20484) df_max = 10 tri_dim = (40960, 3) resl_dim = (61440, 1) c_dim = (1, 2) ef_dim = (1, 20484) sd_dim = (1, 20484) X_dim = (10, 2) coef_dim = (2, 20484) SSE_dim = (1, 20484) finname = datadir("xstatq_11_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, tri_dim=tri_dim, resl_dim=resl_dim, c_dim=c_dim, ef_dim=ef_dim, sd_dim=sd_dim, X_dim=X_dim, coef_dim=coef_dim, SSE_dim=SSE_dim, seed=454, ) foutname = datadir("xstatq_11_OUT.pkl") get_fdr_output(D, foutname) ### test_12 data in-out generation print("test_fdr: test_12 data is generated..") # similar to test_11 + optional input ['mask'] + ['df'] dtype changed # ['t'] : np array, shape (1, 20484), float64 # ['df'] : uint8 # ['k'] : int # ['tri'] : np array, shape (40960, 3), int32 # ['resl'] : np array, shape (61440, 1), float64 # ['c'] : np array, shape (1, 2), float64 # ['ef'] : np array, shape (1, 20484), float64 # ['sd'] : np array, shape (1, 20484), float64 # ['X'] : np array, shape (10, 2), uint8 # ['coef'] : np array, shape (2, 20484), float64 # ['SSE'] : np array, shape (1, 20484), float64 # ['mask'] : np array, shape (20484,), bool t_dim = (1, 20484) df_max = 10 tri_dim = (40960, 3) resl_dim = (61440, 1) c_dim = (1, 2) ef_dim = (1, 20484) sd_dim = (1, 20484) X_dim = (10, 2) coef_dim = (2, 20484) SSE_dim = (1, 20484) mask_dim = 20484 finname = datadir("xstatq_12_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, tri_dim=tri_dim, mask_dim=mask_dim, resl_dim=resl_dim, c_dim=c_dim, ef_dim=ef_dim, sd_dim=sd_dim, X_dim=X_dim, coef_dim=coef_dim, SSE_dim=SSE_dim, seed=455, ) foutname = datadir("xstatq_12_OUT.pkl") get_fdr_output(D, foutname) ### test_13 data in-out generation print("test_fdr: test_13 data is generated..") # similar to test_10 + mask added # ['t'] : np array, shape (1, 20484), float64 # ['df'] : int64 # ['k'] : int # ['tri'] : np array, shape (40960, 3), int32 # ['resl'] : np array, shape (61440, 1), float64 # ['c'] : np array, shape (1, 9), float64 # ['ef'] : np array, shape (1, 20484), float64 # ['sd'] : np array, shape (1, 20484), float64 # ['X'] : np array, shape (20, 9), uint16 # ['coef'] : np array, shape (9, 20484), float64 # ['SSE'] : np array, shape (1, 20484), float64 t_dim = (1, 20484) df_max = 10 tri_dim = (40960, 3) resl_dim = (61440, 1) c_dim = (1, 2) ef_dim = (1, 20484) sd_dim = (1, 20484) mask_dim = 20484 finname = datadir("xstatq_13_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, tri_dim=tri_dim, resl_dim=resl_dim, c_dim=c_dim, mask_dim=mask_dim, ef_dim=ef_dim, sd_dim=sd_dim, seed=453, ) foutname = datadir("xstatq_13_OUT.pkl") get_fdr_output(D, foutname) #### test 14, real data print("test_fdr: test_14 data is generated..") # thickness_n10 data, slm and t_test run prior to fdr realdataf = datadir("thickness_n10.pkl") ifile = open(realdataf, "br") DD = pickle.load(ifile) ifile.close() # run slm M = FixedEffect(DD["M"]) slm = SLM(M, FixedEffect(1)) slm.linear_model(DD["Y"]) D = {} # run t-test t_test(slm) D["t"] = slm.t D["df"] = 10 D["k"] = 1 finname = datadir("xstatq_14_IN.pkl") with open(finname, "wb") as handle: pickle.dump(D, handle, protocol=4) foutname = datadir("xstatq_14_OUT.pkl") get_fdr_output(D, foutname)
def generate_test_data(): pial, mask, age, iq, thickness = load_training_data(n=20) fixed_model = FixedEffect(1) + FixedEffect(age, "age") mixed_model = ( FixedEffect(1) + FixedEffect(age, "age") + MixedEffect(iq, name_ran="iq") + MixedEffect(1, name_ran="Identity") ) variates_2 = np.concatenate( (thickness[:, :, None], np.random.random_sample(thickness.shape)[:, :, None]), axis=2, ) variates_3 = np.concatenate( ( thickness[:, :, None], np.random.rand(thickness.shape[0], thickness.shape[1], 2), ), axis=2, ) # Params 1: No surface, fixed effect. # Params 2: One-tailed mixed with theta/dr changes. # Params 3: With surface. and RFT correction. parameters = [ { "Y": [thickness, variates_2, variates_3], "model": [fixed_model], "contrast": [-age], "correction": [None, "fdr"], "surf": [None], "mask": [mask], "niter": [1], "thetalim": [0.01], "drlim": [0.1], "two_tailed": [True], "cluster_threshold": [0.001], }, { "Y": [thickness], "model": [mixed_model], "contrast": [-age], "correction": ["fdr"], "surf": [None, pial], "mask": [mask], "niter": [1], "thetalim": [0.01, 0.05], "drlim": [0.1, 0.2], "two_tailed": [False], "cluster_threshold": [0.001], }, { "Y": [thickness], "model": [fixed_model, mixed_model], "contrast": [-age], "surf": [pial], "mask": [mask], "correction": [None, ["fdr", "rft"]], "niter": [1], "thetalim": [0.01], "drlim": [0.1], "two_tailed": [True], "cluster_threshold": [0.001, 1.2], }, ] test_num = 0 for params in ParameterGrid(parameters): test_num += 1 slm = SLM( params["model"], params["contrast"], params["surf"], params["mask"], correction=params["correction"], niter=params["niter"], thetalim=params["thetalim"], drlim=params["drlim"], two_tailed=params["two_tailed"], cluster_threshold=params["cluster_threshold"], ) slm.fit(params["Y"]) # Save input/output if isinstance(params["model"], FixedEffect): params["model"] = age[:, None] else: params["model"] = np.concatenate((age[:, None], iq[:, None]), axis=1) dict2pkl(params, "slm", test_num, input=True) slm2files(slm, "slm", test_num, input=False)