def params2files(I, D, test_num): """Converts params to input/output files""" basename = "xstatf" fin_name = datadir(basename + "_" + f"{test_num:02d}" + "_IN.pkl") fout_name = datadir(basename + "_" + f"{test_num:02d}" + "_OUT.pkl") with open(fin_name, "wb") as f: pickle.dump(I, f, protocol=4) with open(fout_name, "wb") as g: pickle.dump(D, g, protocol=4) return
def params2files(rand_dict, O, test_num): """Converts params to input/output files""" # filenames for the input and outpur dict's basename = "xstatsmo" fin_name = datadir(basename + "_" + f"{test_num:02d}" + "_IN.pkl") fout_name = datadir(basename + "_" + f"{test_num:02d}" + "_OUT.pkl") # save input and output data in pickle format with filenames above with open(fin_name, "wb") as f: pickle.dump(rand_dict, f, protocol=4) with open(fout_name, "wb") as g: pickle.dump(O, g, protocol=4) return
def generate_data_test_mesh_edges(): ### test_01 data in-out generation print("test_mesh_edges.py : test_01 data is generated..") # ['tri'] is a 2D numpy array of shape (78, 3), dtype('float64') key_dim = (78, 3) finname = datadir("xstatedg_01_IN.pkl") key_name = "tri" key_dtype = np.float64 seed = 444 surf = generate_random_mesh_edge_data(key_dim, finname, key_name, key_dtype, seed) foutname = datadir("xstatedg_01_OUT.pkl") get_meshedge_output(surf, foutname) ### test_02 data in-out generation print("test_mesh_edges.py : test_02 data is generated..") # ['lat'] is a 2D numpy array of shape (10, 10), dtype('float64') key_dim = (10, 10) finname = datadir("xstatedg_02_IN.pkl") key_name = "lat" key_dtype = np.float64 seed = 445 surf = generate_random_mesh_edge_data(key_dim, finname, key_name, key_dtype, seed) foutname = datadir("xstatedg_02_OUT.pkl") get_meshedge_output(surf, foutname) ### test_03 data in-out generation print("test_mesh_edges.py : test_03 data is generated..") # ['lat'] is a 3D numpy array of shape (10, 10, 10), dtype('int64') key_dim = (10, 10, 10) finname = datadir("xstatedg_03_IN.pkl") key_name = "lat" key_dtype = np.int64 seed = 446 surf = generate_random_mesh_edge_data(key_dim, finname, key_name, key_dtype, seed) foutname = datadir("xstatedg_03_OUT.pkl") get_meshedge_output(surf, foutname) ### test_04 data in-out generation print("test_mesh_edges.py : test_04 data is generated..") # ['tri'] is a 2D numpy array of shape (2044, 3), dtype('uint16') key_dim = (2044, 3) finname = datadir("xstatedg_04_IN.pkl") key_name = "tri" key_dtype = np.uint16 seed = 447 surf = generate_random_mesh_edge_data(key_dim, finname, key_name, key_dtype, seed) foutname = datadir("xstatedg_04_OUT.pkl") get_meshedge_output(surf, foutname)
def slm2files(slm, basename, test_num): """Converts an SLM to its input/output files Parameters ---------- slm : brainstat.stats.SLM SLM object. basename : str Base name for the file. test_num : int Number of the test. """ D = {} D["pval"], D["peak"], D["clus"], D["clusid"] = slm.random_field_theory() save_slm(slm, basename, test_num, input=True) filename = datadir(basename + "_" + f"{test_num:02d}" + "_OUT.pkl") with open(filename, "wb") as f: pickle.dump(D, f, protocol=4)
def dict2pkl(D, basename, test_num, input=True): if "surf" in D and D["surf"] is not None: D["surf"] = { "tri": np.array(get_cells(D["surf"])), "coord": np.array(get_points(D["surf"])).T, } if "_tri" in D: D.pop("_tri") if "_surf" in D and D["_surf"] is not None: D["surf"] = { "tri": np.array(get_cells(D["_surf"])), "coord": np.array(get_points(D["_surf"])), } D.pop("_surf") if input: stage = "IN" else: stage = "OUT" filename = datadir(basename + "_" + f"{test_num:02d}" + "_" + stage + ".pkl") with open(filename, "wb") as f: pickle.dump(D, f, protocol=4)
def generate_data_test_linear_model(): ### test_01 data in-out generation print("test_linear_model: test_01 data is generated..") Y_dim = (43, 43) M_dim = (43, 43) finname = datadir("xlinmod_01_IN.pkl") Y, M = generate_random_test_data(Y_dim, M_dim, finname, seed=444) foutname = datadir("xlinmod_01_OUT.pkl") get_linmod_output(Y, M, foutname) ### test_02 data in-out generation print("test_linear_model: test_02 data is generated..") Y_dim = (62, 7) M_dim = (62, 92) finname = datadir("xlinmod_02_IN.pkl") Y, M = generate_random_test_data(Y_dim, M_dim, finname, seed=445) foutname = datadir("xlinmod_02_OUT.pkl") get_linmod_output(Y, M, foutname) ### test_03 data in-out generation print("test_linear_model: test_03 data is generated..") Y_dim = (54, 64, 76) M_dim = (54, 2) finname = datadir("xlinmod_03_IN.pkl") Y, M = generate_random_test_data(Y_dim, M_dim, finname, seed=446) foutname = datadir("xlinmod_03_OUT.pkl") get_linmod_output(Y, M, foutname) ### test_04 data in-out generation print("test_linear_model: test_04 data is generated..") Y_dim = (69, 41, 5) M_dim = (69, 30) finname = datadir("xlinmod_04_IN.pkl") Y, M = generate_random_test_data(Y_dim, M_dim, finname, seed=447) foutname = datadir("xlinmod_04_OUT.pkl") get_linmod_output(Y, M, foutname) ### test_05 data in-out generation print("test_linear_model: test_05 data is generated..") Y_dim = (81, 1) M_dim = (81, 1) finname = datadir("xlinmod_05_IN.pkl") Y, M = generate_random_test_data(Y_dim, M_dim, finname, seed=448) foutname = datadir("xlinmod_05_OUT.pkl") get_linmod_output(Y, M, foutname) ### test_06 data in-out generation print("test_linear_model: test_06 data is generated..") Y_dim = (93, 41, 57) M_dim = (93, 67) finname = datadir("xlinmod_06_IN.pkl") Y, M = generate_random_test_data(Y_dim, M_dim, finname, seed=448) foutname = datadir("xlinmod_06_OUT.pkl") get_linmod_output(Y, M, foutname) ### test_07 data in-out generation print("test_linear_model: test_07 data is generated..") Y_dim = (40, 46, 21) M_dim = (40, 81) finname = datadir("xlinmod_07_IN.pkl") Y, M = generate_random_test_data(Y_dim, M_dim, finname, seed=449) foutname = datadir("xlinmod_07_OUT.pkl") get_linmod_output(Y, M, foutname) ### test_08 data in-out generation print("test_linear_model: test_08 data is generated..") Y_dim = (93, 43) M_dim = (93, 2) triD = {} triD["tri_min"] = 1 triD["tri_max"] = 42 triD["tri_dim"] = (93, 3) finname = datadir("xlinmod_08_IN.pkl") Y, M, tri = generate_random_test_data(Y_dim, M_dim, finname, seed=450, triD=triD) foutname = datadir("xlinmod_08_OUT.pkl") get_linmod_output(Y, M, foutname, tri=tri) ### test_09 data in-out generation print("test_linear_model: test_09 data is generated..") Y_dim = (98, 69, 60) M_dim = (98, 91) triD = {} triD["tri_min"] = 1 triD["tri_max"] = 68 triD["tri_dim"] = (60, 3) finname = datadir("xlinmod_09_IN.pkl") Y, M, tri = generate_random_test_data(Y_dim, M_dim, finname, seed=451, triD=triD) foutname = datadir("xlinmod_09_OUT.pkl") get_linmod_output(Y, M, foutname, tri=tri) ### test_10 data in-out generation print("test_linear_model: test_10 data is generated..") Y_dim = (49, 27) M_dim = (49, 2) latD = {} latD["lat_min"] = 0 latD["lat_max"] = 2 latD["lat_dim"] = (3, 3, 3) finname = datadir("xlinmod_10_IN.pkl") Y, M, lat = generate_random_test_data(Y_dim, M_dim, finname, seed=452, latD=latD) foutname = datadir("xlinmod_10_OUT.pkl") get_linmod_output(Y, M, foutname, lat=lat) ### test_11 data in-out generation print("test_linear_model: test_11 data is generated..") Y_dim = (45, 27, 3) M_dim = (45, 7) latD = {} latD["lat_min"] = 0 latD["lat_max"] = 2 latD["lat_dim"] = (3, 3, 3) finname = datadir("xlinmod_11_IN.pkl") Y, M, lat = generate_random_test_data(Y_dim, M_dim, finname, seed=453, latD=latD) foutname = datadir("xlinmod_11_OUT.pkl") get_linmod_output(Y, M, foutname, lat=lat) ### test_12 data in-out generation print("test_linear_model: test_12 data is generated..") # this is real data, save manually.." realdataf = datadir("thickness_n10.pkl") ifile = open(realdataf, "br") D = pickle.load(ifile) D["M"] = D["M"].T ifile.close() Y = D["Y"] M = D["M"] tri = D["tri"] finname = datadir("xlinmod_12_IN.pkl") with open(finname, "wb") as handle: pickle.dump(D, handle, protocol=4) foutname = datadir("xlinmod_12_OUT.pkl") get_linmod_output(Y, M, foutname, tri=tri) ### test_13: real in data shuffled print("test_linear_model: test_13 data is generated..") realdataf = datadir("thickness_n10.pkl") ifile = open(realdataf, "br") D = pickle.load(ifile) D["M"] = D["M"].T ifile.close() Y = D["Y"] np.random.seed(seed=454) np.random.shuffle(Y) M = D["M"] tri = D["tri"] finname = datadir("xlinmod_13_IN.pkl") with open(finname, "wb") as handle: pickle.dump(D, handle, protocol=4) foutname = datadir("xlinmod_13_OUT.pkl") get_linmod_output(Y, M, foutname, tri=tri) ### test_14: real in data shuffled print("test_linear_model: test_14 data is generated..") realdataf = datadir("thickness_n10.pkl") ifile = open(realdataf, "br") Din = pickle.load(ifile) ifile.close() Y = Din["Y"] M = Din["M"].T tri = Din["tri"] np.random.seed(seed=455) np.random.shuffle(Y) np.random.seed(seed=456) np.random.shuffle(tri) # save D = {} D["Y"] = Y D["M"] = M D["tri"] = tri finname = datadir("xlinmod_14_IN.pkl") with open(finname, "wb") as handle: pickle.dump(D, handle, protocol=4) foutname = datadir("xlinmod_14_OUT.pkl") get_linmod_output(Y, M, foutname, tri=tri) ### test_15: real in data shuffled and is manually extended print("test_linear_model: test_15 data is generated..") realdataf = datadir("thickness_n10.pkl") ifile = open(realdataf, "br") Din = pickle.load(ifile) ifile.close() Y = Din["Y"] A = Y.copy() # extend Y np.random.seed(seed=457) np.random.shuffle(Y) Y = np.concatenate((A, Y), axis=0) # (20, 20484) # generate M manually a = np.ones((20, 1)) np.random.seed(seed=456) b = np.random.randint(22, 51, size=(20, 1)) np.random.seed(seed=457) c = np.random.randint(0, 2, size=(20, 1)) np.random.seed(seed=458) d = np.random.randint(0, 2, size=(20, 1)) e = np.zeros((20, 1)) f = np.ones((20, 1)) g = np.zeros((20, 1)) h = np.zeros((20, 1)) np.random.seed(seed=459) i = np.random.randint(10120, 22030, size=(20, 1)) M = np.concatenate((a, b, c, d, e, f, g, h, i), axis=1) # (20, 9) # get tri from real data tri = Din["tri"] # (40960, 3) # save D = {} D["Y"] = Y D["M"] = M D["tri"] = tri finname = datadir("xlinmod_15_IN.pkl") with open(finname, "wb") as handle: pickle.dump(D, handle, protocol=4) foutname = datadir("xlinmod_15_OUT.pkl") get_linmod_output(Y, M, foutname, tri=tri)
def test_run_all(test_number): infile = datadir("xstatresl_" + f"{test_number:02d}" + "_IN.pkl") expfile = datadir("xstatresl_" + f"{test_number:02d}" + "_OUT.pkl") dummy_test(infile, expfile)
def generate_data_test_fdr(): ### test_01 data in-out generation print("test_fdr: test_01 data is generated..") # random data shape matching a real-data set # ['t'] : np array, shape (1, 64984), float64 # ['df'] : int # ['k'] : int t_dim = (1, 64984) df_max = 64984 finname = datadir("xstatq_01_IN.pkl") D = generate_random_fdr_data(t_dim, df_max, finname, seed=444) foutname = datadir("xstatq_01_OUT.pkl") get_fdr_output(D, foutname) ### test_02 data in-out generation print("test_fdr: test_02 data is generated..") # random data # ['t'] : np array, shape (1, 9850), float64 # ['df'] : int # ['k'] : int t_dim = (1, 9850) df_max = 1000 finname = datadir("xstatq_02_IN.pkl") D = generate_random_fdr_data(t_dim, df_max, finname, seed=445) foutname = datadir("xstatq_02_OUT.pkl") get_fdr_output(D, foutname) ### test_03 data in-out generation print("test_fdr: test_03 data is generated..") # similar to test_02, shapes/values of slm['t'] and slm['df'] manipulated # ['t'] : np array, shape (1, 2139), float64 # ['df'] : int # ['k'] : int t_dim = (1, 2139) df_max = 2000 k = 3 finname = datadir("xstatq_03_IN.pkl") D = generate_random_fdr_data(t_dim, df_max, finname, k=k, seed=446) foutname = datadir("xstatq_03_OUT.pkl") get_fdr_output(D, foutname) ### test_04 data in-out generation print("test_fdr: test_04 data is generated..") # similar to test_02 + optional input ['mask'] # ['t'] : np array, shape (1, 2475), float64 # ['df'] : int # ['k'] : int # ['mask'] : np array, shape (2475,), bool t_dim = (1, 2475) df_max = 1500 finname = datadir("xstatq_04_IN.pkl") mask_dim = 2475 D = generate_random_fdr_data(t_dim, df_max, finname, mask_dim=mask_dim, seed=447) foutname = datadir("xstatq_04_OUT.pkl") get_fdr_output(D, foutname) ### test_05 data in-out generation print("test_fdr: test_05 data is generated..") # similar to test_02 + optional input slm['dfs'] # ['t'] : np array, shape (1, 1998), float64 # ['df'] : int # ['k'] : int # ['dfs'] : np array, shape (1, 1998), int64 t_dim = (1, 1998) df_max = 4000 dfs_max = 1997 finname = datadir("xstatq_05_IN.pkl") D = generate_random_fdr_data(t_dim, df_max, finname, dfs_max=dfs_max, seed=448) foutname = datadir("xstatq_05_OUT.pkl") get_fdr_output(D, foutname) ### test_06 data in-out generation print("test_fdr: test_06 data is generated..") # similar to test_02 + optional inputs slm['dfs'] and ['mask'] # ['t'] : np array, shape (1, 3328), float64 # ['df'] : np array, shape (1, 1), int64 # ['k'] : int # ['dfs'] : np array, shape (1, 3328), int64 # ['mask'] : np array, shape (3328,), bool t_dim = (1, 3328) df_max = 10000 k = 2 dfs_max = 3328 mask_dim = 3328 finname = datadir("xstatq_06_IN.pkl") D = generate_random_fdr_data(t_dim, df_max, finname, k=k, dfs_max=dfs_max, mask_dim=mask_dim, seed=449) foutname = datadir("xstatq_06_OUT.pkl") get_fdr_output(D, foutname) ### test_07 data in-out generation print("test_fdr: test_07 data is generated..") # similar to test_02 + optional inputs slm['dfs'], ['mask'] and ['tri'] # ['t'] : np array, shape (1, 9512), float64 # ['df'] : int # ['k'] : int # ['dfs'] : np array, shape (1, 9512), int64 # ['mask'] : np array, shape (9512,), bool # ['tri'] : np array, shape (1724, 3), int64 t_dim = (1, 9512) df_max = 5000 dfs_max = 9511 mask_dim = 9512 tri_dim = (1724, 3) finname = datadir("xstatq_07_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, dfs_max=dfs_max, mask_dim=mask_dim, tri_dim=tri_dim, seed=450, ) foutname = datadir("xstatq_07_OUT.pkl") get_fdr_output(D, foutname) ### test_08 data in-out generation print("test_fdr: test_08 data is generated..") # similar to test_02 + optional inputs slm['dfs'], slm['tri'] and slm['resl'] # ['t'] : np array, shape (1, 1520), float64 # ['df'] : int # ['k'] : int # ['dfs'] : np array, shape (1, 1520), int64 # ['tri'] : np array, shape (4948, 3), int64 # ['resl'] : np array, shape (1520, 1), float64 t_dim = (1, 1520) df_max = 5000 k = 5 dfs_max = 9 tri_dim = (4948, 3) resl_dim = (1520, 1) finname = datadir("xstatq_08_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, k=k, dfs_max=dfs_max, tri_dim=tri_dim, resl_dim=resl_dim, seed=451, ) foutname = datadir("xstatq_08_OUT.pkl") get_fdr_output(D, foutname) ### test_09 data in-out generation print("test_fdr: test_09 data is generated..") # similar to test_08 + values/shapes of input params changed + # additional input slm['du'] (non-sense for _fdr) # ['t'] : np array, shape (1, 4397), float64 # ['df'] : int # ['k'] : int # ['tri'] : np array, shape (2734, 3), int64 # ['resl'] : np array, shape (8199, 1), float64 # ['dfs'] : np array, shape (1, 4397), float64 # ['du'] : int t_dim = (1, 14397) df_max = 1 dfs_max = 2 tri_dim = (2734, 3) resl_dim = (8199, 1) # du = 9 finname = datadir("xstatq_09_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, dfs_max=dfs_max, tri_dim=tri_dim, resl_dim=resl_dim, # du = du, seed=452, ) foutname = datadir("xstatq_09_OUT.pkl") get_fdr_output(D, foutname) ### test_10 data in-out generation print("test_fdr: test_10 data is generated..") # similar to test_08 + + values/shapes of input params changed + additional # input slm['du'], slm['c'], slm['ef'], and slm['sd'] (non-sense for _fdr) # ['t'] : np array, shape (1, 20484), float64 # ['df'] : int # ['k'] : int # ['tri'] : np array, shape (40960, 3), int32 # ['resl'] : np array, shape (61440, 1), float64 # ['c'] : np array, shape (1, 2), float64 # ['ef'] : np array, shape (1, 20484), float64 # ['sd'] : np array, shape (1, 20484), float64 t_dim = (1, 20484) df_max = 10 tri_dim = (40960, 3) resl_dim = (61440, 1) c_dim = (1, 2) ef_dim = (1, 20484) sd_dim = (1, 20484) finname = datadir("xstatq_10_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, tri_dim=tri_dim, resl_dim=resl_dim, c_dim=c_dim, ef_dim=ef_dim, sd_dim=sd_dim, seed=453, ) foutname = datadir("xstatq_10_OUT.pkl") get_fdr_output(D, foutname) ### test_11 data in-out generation print("test_fdr: test_11 data is generated..") # similar to test_08 + additional input ['c'], ['ef'], ['sd'], ['X'], # and ['coef'], ['SSE'] (non-sense for _fdr) # ['t'] : np array, shape (1, 20484), float64 # ['df'] : int # ['k'] : int # ['tri'] : np array, shape (40960, 3), int32 # ['resl'] : np array, shape (61440, 1), float64 # ['c'] : np array, shape (1, 2), float64 # ['ef'] : np array, shape (1, 20484), float64 # ['sd'] : np array, shape (1, 20484), float64 # ['X'] : np array, shape (10, 2), float64 # ['coef'] : np array, shape (2, 20484), float64 # ['SSE'] : np array, shape (1, 20484), float64 t_dim = (1, 20484) df_max = 10 tri_dim = (40960, 3) resl_dim = (61440, 1) c_dim = (1, 2) ef_dim = (1, 20484) sd_dim = (1, 20484) X_dim = (10, 2) coef_dim = (2, 20484) SSE_dim = (1, 20484) finname = datadir("xstatq_11_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, tri_dim=tri_dim, resl_dim=resl_dim, c_dim=c_dim, ef_dim=ef_dim, sd_dim=sd_dim, X_dim=X_dim, coef_dim=coef_dim, SSE_dim=SSE_dim, seed=454, ) foutname = datadir("xstatq_11_OUT.pkl") get_fdr_output(D, foutname) ### test_12 data in-out generation print("test_fdr: test_12 data is generated..") # similar to test_11 + optional input ['mask'] + ['df'] dtype changed # ['t'] : np array, shape (1, 20484), float64 # ['df'] : uint8 # ['k'] : int # ['tri'] : np array, shape (40960, 3), int32 # ['resl'] : np array, shape (61440, 1), float64 # ['c'] : np array, shape (1, 2), float64 # ['ef'] : np array, shape (1, 20484), float64 # ['sd'] : np array, shape (1, 20484), float64 # ['X'] : np array, shape (10, 2), uint8 # ['coef'] : np array, shape (2, 20484), float64 # ['SSE'] : np array, shape (1, 20484), float64 # ['mask'] : np array, shape (20484,), bool t_dim = (1, 20484) df_max = 10 tri_dim = (40960, 3) resl_dim = (61440, 1) c_dim = (1, 2) ef_dim = (1, 20484) sd_dim = (1, 20484) X_dim = (10, 2) coef_dim = (2, 20484) SSE_dim = (1, 20484) mask_dim = 20484 finname = datadir("xstatq_12_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, tri_dim=tri_dim, mask_dim=mask_dim, resl_dim=resl_dim, c_dim=c_dim, ef_dim=ef_dim, sd_dim=sd_dim, X_dim=X_dim, coef_dim=coef_dim, SSE_dim=SSE_dim, seed=455, ) foutname = datadir("xstatq_12_OUT.pkl") get_fdr_output(D, foutname) ### test_13 data in-out generation print("test_fdr: test_13 data is generated..") # similar to test_10 + mask added # ['t'] : np array, shape (1, 20484), float64 # ['df'] : int64 # ['k'] : int # ['tri'] : np array, shape (40960, 3), int32 # ['resl'] : np array, shape (61440, 1), float64 # ['c'] : np array, shape (1, 9), float64 # ['ef'] : np array, shape (1, 20484), float64 # ['sd'] : np array, shape (1, 20484), float64 # ['X'] : np array, shape (20, 9), uint16 # ['coef'] : np array, shape (9, 20484), float64 # ['SSE'] : np array, shape (1, 20484), float64 t_dim = (1, 20484) df_max = 10 tri_dim = (40960, 3) resl_dim = (61440, 1) c_dim = (1, 2) ef_dim = (1, 20484) sd_dim = (1, 20484) mask_dim = 20484 finname = datadir("xstatq_13_IN.pkl") D = generate_random_fdr_data( t_dim, df_max, finname, tri_dim=tri_dim, resl_dim=resl_dim, c_dim=c_dim, mask_dim=mask_dim, ef_dim=ef_dim, sd_dim=sd_dim, seed=453, ) foutname = datadir("xstatq_13_OUT.pkl") get_fdr_output(D, foutname) #### test 14, real data print("test_fdr: test_14 data is generated..") # thickness_n10 data, slm and t_test run prior to fdr realdataf = datadir("thickness_n10.pkl") ifile = open(realdataf, "br") DD = pickle.load(ifile) ifile.close() # run slm M = FixedEffect(DD["M"]) slm = SLM(M, FixedEffect(1)) slm.linear_model(DD["Y"]) D = {} # run t-test t_test(slm) D["t"] = slm.t D["df"] = 10 D["k"] = 1 finname = datadir("xstatq_14_IN.pkl") with open(finname, "wb") as handle: pickle.dump(D, handle, protocol=4) foutname = datadir("xstatq_14_OUT.pkl") get_fdr_output(D, foutname)