def prepare_GLM(baseline_stats=False):
if baseline_stats:
curr_stats = CV_sw_test()
else:
curr_stats = GLM_delay_stats()
all_sess_list = []
path_list = []
reg_list = []
dpath = align.get_root_path()
for (pid, path) in enumerate(align.traverse(dpath)):
print(path)
# SU_ids = []
trial_FR = None
trials = None
if not os.path.isfile(os.path.join(path, "su_id2reg.csv")):
continue
done_read = False
while not done_read:
try:
with h5py.File(os.path.join(path, "FR_All.hdf5"), "r") as ffr:
# print(list(ffr.keys()))
if not "SU_id" in ffr.keys():
done_read = True
print("missing su_id key in path ", path)
continue
# dset = ffr["SU_id"]
# SU_ids = np.array(dset, dtype="uint16")
dset = ffr["FR_All"]
trial_FR = np.array(dset, dtype="double")
dset = ffr["Trials"]
trials = np.array(dset, dtype="double").T
done_read = True
except OSError:
print("h5py read error handled")
if trials is None:
continue
with open(os.path.join(path, "su_id2reg.csv")) as csvfile:
l = list(csv.reader(csvfile))[1:]
suid_reg = [list(i) for i in zip(*l)]
(perf_desc, perf_code, welltrain_window, correct_resp) = align.judgePerformance(trials)
if perf_code != 3:
continue
curr_stats.processGLMStats(trial_FR, trials, welltrain_window, correct_resp, path, pid)
(onesession, onepath) = curr_stats.getFeatures()
all_sess_list.append(onesession)
path_list.extend(onepath)
reg_list.extend(suid_reg[1])
return (all_sess_list, path_list, reg_list)
dset = fe["trials"]
trials = np.array(dset, dtype="int32")
return trials
if __name__ == "__main__":
# %% setup
FR_Th = 1.0
regionMatched = []
regionL = align.getRegionList()
# conversion = []
# %% main loop
for path in align.traverse("K:/neupix/DataSum/"):
sampSel = []
pairSel = []
if not os.path.isfile(os.path.join(path, "selectivity.hdf5")):
if not os.path.isfile(os.path.join(path, "NOSU")):
print("missing one selectivity file, path is", path)
input("press Enter to continue")
continue
with h5py.File(os.path.join(path, "selectivity.hdf5"), "r") as fe:
dset = fe["sampleSel"]
sampSel = np.transpose(np.array(dset, dtype="double"))
dset = fe["pairSel"]
pairSel = np.transpose(np.array(dset, dtype="double"))
import re
import numpy as np
import pandas as pd
import su_region_align as align
import selectivity as zpy
trials = []
learningTrials = []
paths = []
ids = []
regions = []
unlabeledRecord = []
(regionL, non) = align.getRegionList(cmp=False)
for path in align.traverse(r"/home/zx/neupix/wyt/DataSum"):
if not os.path.isfile(os.path.join(path, "events.hdf5")):
print("missing one event file, path: ", path)
continue
print(path)
# judge performance first
os.chdir(path)
trials = np.empty([0])
with h5py.File(os.path.join(path, "events.hdf5"), 'r') as fe:
dset = fe['trials']
trials = np.array(dset, dtype='int32')
inWindow = align.judgePerformance(trials)
if np.count_nonzero(inWindow == 2) >= 40:
def run_tca(trial_target, non_neg=True, sep_blocks=False, epoc=[], effect=[]):
ntrialsCount = []
all_sess_list = []
reg_list = []
for path in align.traverse(align.get_root_path()):
print(path)
# SU_ids = []
trial_FR = []
trials = []
done_read = False
while not done_read:
try:
with h5py.File(os.path.join(path, "FR_All.hdf5"), "r") as ffr:
# print(list(ffr.keys()))
if not "SU_id" in ffr.keys():
print("missing su_id key in path ", path)
done_read = True
continue
# dset = ffr["SU_id"]
# SU_ids = np.array(dset, dtype="uint16")
dset = ffr["FR_All"]
trial_FR = np.array(dset, dtype="double")
dset = ffr["Trials"]
trials = np.array(dset, dtype="double").T
done_read = True
except OSError:
print("h5py read error handled")
if not os.path.isfile(os.path.join(path, "su_id2reg.csv")):
continue
### It is the logic of the matlab code that suid to brain region is in identical sequence
suid_reg = []
with open(os.path.join(path, "su_id2reg.csv")) as csvfile:
l = list(csv.reader(csvfile))[1:]
suid_reg = [list(i) for i in zip(*l)]
# print(res)
(perf_desc, perf_code, inWindow,
correct_resp) = align.judgePerformance(trials)
# toReturn.extend(["wellTrained", 3, correctResp,welltrain_window])
if perf_code != 3:
continue
ntrialsCount.append(trials.shape[0])
if trials.shape[0] < trial_target:
continue
if sep_blocks:
(aligned_all,
matched_index) = rearrange_sep_block(trials, trial_target)
else:
(aligned_all,
matched_index) = rearrange_block(trials, trial_target)
if aligned_all:
merged = rearrange_row(trials, trial_FR)
# onesession=merged[:,matched_index,:].reshape((merged.shape[0],-1))
### TODO
# select region here
if epoc and effect:
su_sel = isin(suid_reg[1], epoc, effect)
if not any(su_sel):
continue
reg_list.extend(itertools.compress(suid_reg[1], su_sel))
merged = merged[su_sel, :, :]
else:
reg_list.extend(suid_reg[1])
onesession = merged[:, matched_index, :]
all_sess_list.append(onesession)
all_sess_arr = np.concatenate(tuple(all_sess_list), axis=0)
### Aligned activity profile of all neurons with brain region labeling can be saved HERE
# np.save('all_sess_arr_160.npy',all_sess_arr)
# np.save('reg_list_160.npy',np.array(reg_list))
#breakpoint()
all_sess_arr = normalize(all_sess_arr)
opti_param = []
suid_reg_arr = np.array(reg_list, dtype="|S10")
sep_str = "sepblock_" if sep_blocks else "consec_"
opti_str = 'ncp_bcd_' if non_neg else 'cp_als'
for R in range(5, 16):
(U, V, sim) = tca(all_sess_arr,
non_neg,
R,
prefix=sep_str,
max_iter=1000,
epoc=epoc,
effect=effect)
np.savez_compressed(
opti_str + sep_str + "tensor_comp_trial" +
str(all_sess_arr.shape[1]) + "_R" + str(R) + "_" + epoc + "_" +
effect + ".npz",
SU=U.factors.factors[0],
cross_trial=U.factors.factors[1],
in_trial=U.factors.factors[2],
reg_list=suid_reg_arr,
)
opti_param.append([R, U.obj, V.obj, sim])
np.save(
opti_str + sep_str + "nonneg_trials_" + epoc + "_" + effect + "_" +
str(trial_target) + "_opti_params.npy",
np.array(opti_param),
)
def get_dataset(denovo):
features_per_su = []
avails = []
reg_list = []
if denovo:
dpath = align.get_root_path()
for path in align.traverse(dpath):
print(path)
# SU_ids = []
trial_FR = None
trials = None
if not os.path.isfile(os.path.join(path, "su_id2reg.csv")):
continue
done_read = False
while not done_read:
try:
with h5py.File(os.path.join(path, "FR_All.hdf5"), "r") as ffr:
# print(list(ffr.keys()))
if not "SU_id" in ffr.keys():
done_read = True
print("missing su_id key in path ", path)
continue
# dset = ffr["SU_id"]
# SU_ids = np.array(dset, dtype="uint16")
dset = ffr["FR_All"]
trial_FR = np.array(dset, dtype="double")
dset = ffr["Trials"]
trials = np.array(dset, dtype="double").T
done_read = True
except OSError:
print("h5py read error handled")
if trials is None:
continue
(_perf_desc, perf_code, welltrain_window, correct_resp,) = align.judgePerformance(trials)
if perf_code != 3:
continue
suid_reg = []
with open(os.path.join(path, "su_id2reg.csv")) as csvfile:
l = list(csv.reader(csvfile))[1:]
suid_reg = [list(i) for i in zip(*l)]
currStats = ctd_stats()
currStats.processCTDStats(trial_FR, trials, welltrain_window, correct_resp)
(feat, avail) = currStats.get_features()
features_per_su.extend(feat)
avails.extend(avail)
reg_list.extend(suid_reg[1])
### DEBUG in small subsets
# if len(features_per_su)>50:
# break
### save to npz file
# np.savez_compressed(f"ctd_prev_stats.npz", features_per_su=features_per_su, reg_list=reg_list, avails=avails)
### load from npz file
else:
fstr = np.load("ctd_prev_stats.npz", allow_pickle=True)
features_per_su = fstr["features_per_su"].tolist()
reg_list = fstr["reg_list"].tolist()
avails = fstr['avails']
return (features_per_su, reg_list, avails)