def statsmodel_solution(animal, day, maxlag=5):
import h5py
import numpy as np
from utils_loading import encode_to_filename
import matplotlib.pyplot as plt
from statsmodels.tsa.stattools import grangercausalitytests
folder = "/Volumes/DATA_01/NL/layerproject/processed/"
hf = h5py.File(encode_to_filename(folder, animal, day), 'r')
dff = np.array(hf['dff'])
NEUR = 'ens'
# Critical neuron pair vs general neuron gc
if NEUR == 'ens':
rois = dff[hf['ens_neur']]
elif NEUR == 'neur':
rois = dff[hf['nerden']]
else:
rois = dff[NEUR]
# rois: N * T
gcs_val = np.zeros((rois.shape[0], rois.shape[0], maxlag))
tests = ['ssr_ftest', 'ssr_chi2test', 'lrtest','params_ftest']
p_vals = {t:np.zeros((rois.shape[0], rois.shape[0], maxlag)) for t in tests}
for i in range(rois.shape[0]):
for j in range(rois.shape[0]):
res = grangercausalitytests(rois[[j, i]].T, maxlag)
for k in res:
test, reg = res[k]
ssrEig = reg[0].ssr
ssrBeid = reg[1].ssr
gcs_val[i, j, k-1] = np.log(ssrEig / ssrBeid)
for t in tests:
p_vals[t][i, j, k-1] = test[t][1]
#TODO: USE LOG stats of two ssrs
return gcs_val, p_vals
def digitize_calcium_all(folder, groups, source, ns, nproc=1):
# TODO: ADD OPTION TO PASS IN A LIST OF METHODS FOR COMPARING THE PLOTS!
"""Calculates Peak Timing and Stores them in csvs for all animal sessions in groups located in folder."""
processed = os.path.join(folder, 'CaBMI_analysis/processed')
logfolder = os.path.join(processed, 'log')
if not os.path.exists(logfolder):
os.makedirs(logfolder)
all_files = parse_group_dict(processed, groups, 'all')
print(all_files)
for n in ns:
if nproc == 0:
nproc = mp.cpu_count()
if nproc == 1:
for animal in all_files:
for day in (all_files[animal]):
print(animal, day)
hf = encode_to_filename(processed, animal, day)
digitize_calcium(hf, source, n)
else:
p = mp.Pool(nproc)
p.starmap_async(digitize_calcium_by_animal,
[(processed, animal, all_files[animal], source, n)
for animal in all_files])
with open("dCalcium_n_{}.txt".format(n)) as f:
f.write("done")
def get_roi_type(processed, animal, day):
# TODO: fix problems when ens becomes None
rois = None
if isinstance(processed, str):
hfile = h5py.File(encode_to_filename(processed, animal, day), 'r')
else:
hfile = processed
N = hfile['C'].shape[0]
rois = np.full(N, "D", dtype="U2")
nerden = np.array(hfile['nerden'])
redlabel = np.array(hfile['redlabel'])
ens_neur = np.array(hfile['ens_neur'])
ens_neur = ens_neur[~np.isnan(ens_neur)].astype(np.int)
e2_neur = None
if 'e2_neur' in hfile:
temp = np.array(hfile['e2_neur'])
if ~np.any(np.isnan(temp)):
e2_neur = temp.astype(np.int)
if isinstance(processed, str):
hfile.close()
rois[nerden & ~redlabel] = 'IG'
rois[nerden & redlabel] = 'IR'
if e2_neur is not None:
rois[ens_neur] = 'E1'
rois[e2_neur] = 'E2'
else:
rois[ens_neur] = 'E'
return rois
def deconvolve_reconvolve_single_session_test(processed,
animal,
day,
randN=None,
savePlot=None,
showPlot=True,
**kwargs):
with h5py.File(encode_to_filename(processed, animal, day), 'r') as hfile:
roi_type = get_roi_type(hfile, animal, day)
e_sel = np.isin(roi_type, ['E1', 'E2', 'E'])
ind_sel = np.isin(roi_type, ['IR', 'IG'])
nerden = np.array(hfile['nerden'])
dff = np.array(hfile['dff'])
dff_e = dff[e_sel, :]
dff_ind = dff[ind_sel, :]
dff_all = np.vstack([dff_e, dff_ind])
if randN is not None:
rinds = np.arange(dff_all.shape[0])
np.random.shuffle(rinds)
dff_all = dff_all[rinds[:randN]]
reconvs = np.full_like(dff_all, np.nan)
all_sn = np.full(dff_all.shape[0], np.nan)
recleans = np.full_like(dff_all, np.nan)
valid_selector = np.full(dff_all.shape[0], 1, dtype=bool)
for i in range(dff_all.shape[0]):
print(i)
try:
c, c1, sp, sn, reconv, reconv_clean = deconvolve_reconvolve_test(
dff_all[i], tag=str(i), show=False, save=None, **kwargs)
reconvs[i] = reconv
all_sn[i] = sn
recleans[i] = reconv_clean
except:
valid_selector[i] = False
TAG = f'{animal}_{day}_dff_reconvolve'
for nthres in [0, 0.5, 1]:
corrs_clean = noise_free_corr(dff_all[valid_selector],
recleans[valid_selector],
all_sn[valid_selector],
noise_thres=nthres,
tag=TAG,
save=savePlot,
show=showPlot)
# corrs = noise_free_corr(dff_all[valid_selector], reconvs[valid_selector], all_sn[valid_selector],
# None, noise_thres=nthres, tag=TAG, save=savePlot,
# show=showPlot)
ksps = test_distribution(dff_all[valid_selector],
reconvs[valid_selector],
tag1='dff',
tag2='reconvolve',
alltag=TAG,
save=savePlot,
show=showPlot)
return corrs_clean, ksps
def raw_activity_tuning_single_session(folder,
animal,
day,
i,
window=3000,
itype='dff',
metric='raw',
zcap=None):
hf = encode_to_filename(processed, animal, day)
if not os.path.exists(hf):
print("Not found:, ", hf)
with h5py.File(hf, 'r') as fp:
S = np.array(fp[itype])
# TODO: maybe include trial activity tuning
#array_hit, array_miss = np.array(fp['array_t1']), np.array(fp['array_miss'])
ens_neur = np.array(fp['ens_neur'])
e2_neur = ens_neur[fp['e2_neur']] if 'e2_neur' in fp else None
redlabel, nerden = np.array(fp['redlabel']), np.array(fp['nerden'])
rois = get_roi_type(fp, animal, day)
if metric == 'mean' and zcap is not None:
zscoreS = zscore(S, axis=1)
if zcap != -1:
zscoreS = np.minimum(zscoreS, np.full_like(S, zcap))
else:
zscoreS = S
N = S.shape[0]
nsessions = S.shape[1] // window
remainder = S.shape[1] - nsessions * window
Sfirst = (zscoreS[:, :nsessions * window]).reshape(
(S.shape[0], nsessions, window), order='C')
avg_S = np.nanmean(Sfirst, axis=2)
if remainder > 0:
Sremain = zscoreS[:, nsessions * window:]
avg_S = np.concatenate(
(avg_S, np.nanmean(Sremain, keepdims=True, axis=1)), axis=1)
# N, sw, st = probeW.shape[0], probeW.shape[1], probeT.shape[1]
# ROI_type
sw = nsessions + (1 if remainder else 0)
# DF Window
results = [
np.tile(np.arange(sw), N),
np.repeat(rois, sw),
np.repeat(np.arange(N), sw),
avg_S.ravel(order='C'),
np.full(N * sw, animal[:2]),
np.full(N * sw, animal),
np.full(N * sw, day),
np.full(N * sw, i + 1)
]
print(animal, day, 'done')
return results
def __init__(self,
folder,
animal,
day,
sec_var='',
lag=2,
method='te-extended',
out=None):
"""
method: str
xc, by finding the peak in the cross-correlogram between the two time series
mi, by finding the lag with the largest Mutual Information
gc, by computing the Granger Causality, based on: C.W.J. Granger, Investigating Causal Relations by Econometric Models and Cross-Spectral Methods , Econometrica, 1969
te-extended, by computing GTE as defined above.
TE and GTE without binning:
te-binless-Leonenko, based on: L.F. Kozachenko and N.N. Leonenko, 1987
te-binless-Kraskov, based on: A. Kraskov et al., 2004
te-binless-Frenzel, based on: S. Frenzel and B. Pompe, 2007
te-symbolic (experimental) based on: M. Staniek and K. Lehnertz, 2008.
out: str
path of the root output directory, if left None, default to {folder}/utils/FC/{method}/
"""
if out is None:
out = os.path.join(folder, 'utils/FC/')
self.out_path = os.path.join(out, f'te-package_' + method, animal, day,
'')
if not os.path.exists(self.out_path):
os.makedirs(self.out_path)
self.folder = folder
self.animal = animal
self.day = day
self.parameters = {
"AutoConditioningLevelQ": True,
'AutoBinNumberQ': True,
'SourceMarkovOrder': lag,
'TargetMarkovOrder': lag,
'StartSampleIndex': 2
} # update conditioning level for gte
self.exp_file = h5py.File(
encode_to_filename(os.path.join(folder, 'processed'), animal, day),
'r')
self.blen = self.exp_file.attrs['blen']
self.method = method
def get_general_GC_distribution(folder, resamp=0.1, concat=True, save=True):
""" Plots distplot for the entire ens-indirect GC distribution
Input:
folder: root folder
"""
fc = os.path.join(folder, "utils/FC")
rsampTag = f"_resamp{resamp}" if resamp else ""
alldistf = os.path.join(fc, f"gc_ens_to_I_order2_alldist{rsampTag}.npy")
if os.path.exists(alldistf):
assert concat, "fast load only supports concatenated format"
return np.load(alldistf)
utils = os.path.join(folder, 'utils')
processed = os.path.join(folder, 'processed')
alles = []
for animal in get_all_animals(processed):
animal_path = os.path.join(processed, animal)
for day in get_animal_days(animal_path):
print(animal, day)
roi_type = get_roi_type(processed, animal, day)
pfname = encode_to_filename(utils,
animal,
day,
hyperparams='granger')
GC_inds, ens_to_I = get_ens_to_indirect_GC(pfname, roi_type)
ens_to_I_rav = ens_to_I.ravel()
if resamp is not None:
resampsize = int(resamp * len(ens_to_I_rav))
# TODO: maybe fix seeds?
rand_inds = sample_without_replacement(
n_population=len(ens_to_I_rav), n_samples=resampsize)
ens_to_I_rav = ens_to_I_rav[rand_inds]
alles.append(ens_to_I_rav)
concats = np.concatenate(alles)
if save:
np.save(alldistf, concats)
if concat:
return concats
else:
return np.vstack(alles)
def calcium_to_peak_times_all(folder, groups, low=1, high=20):
# TODO: ADD OPTION TO PASS IN A LIST OF METHODS FOR COMPARING THE PLOTS!
"""Calculates Peak Timing and Stores them in csvs for all animal sessions in groups located in folder."""
processed = os.path.join(folder, 'CaBMI_analysis/processed')
if groups == '*':
all_files = get_PTIT_over_days(processed)
else:
all_files = {
g: parse_group_dict(processed, groups[g], g)
for g in groups.keys()
}
print(all_files)
for group in all_files:
group_dict = all_files[group]
for animal in group_dict:
for day in (group_dict[animal]):
print(animal, day)
hf = encode_to_filename(processed, animal, day)
calcium_to_peak_times(hf, low, high)
def nitime_solution(animal, day, cutoff=True):
# TODO: add support for custom freq bands
import h5py
import nitime
import numpy as np
import nitime.analysis as nta
from utils_loading import encode_to_filename
from nitime.viz import drawmatrix_channels
import matplotlib.pyplot as plt
folder = "/Volumes/DATA_01/NL/layerproject/processed/"
hf = h5py.File(encode_to_filename(folder, animal, day), 'r')
dff = np.array(hf['dff'])
NEUR = 'ens'
# Critical neuron pair vs general neuron gc
if NEUR == 'ens':
rois = dff[hf['ens_neur']]
elif NEUR == 'neur':
rois = dff[hf['nerden']]
else:
rois = dff[NEUR]
rois_ts = nitime.TimeSeries(rois, sampling_interval=1 / hf.attrs['fr'])
G = nta.GrangerAnalyzer(rois_ts)
if cutoff:
sel = np.where(G.frequencies < hf.attrs['fr'])[0]
caus_xy = G.causality_xy[:, :, sel]
caus_yx = G.causality_yx[:, :, sel]
caus_sim = G.simultaneous_causality[:, :, sel]
else:
caus_xy = G.causality_xy
caus_yx = G.causality_yx
caus_sim = G.simultaneous_causality
g1 = np.mean(caus_xy, -1)
g2 = np.mean(caus_yx, -1)
g3 = np.mean(caus_sim, -1)
g4 = g1-g2
fig03 = drawmatrix_channels(g1, ['E11', 'E12', 'E21', 'E22'], size=[10., 10.], color_anchor = 0)
plt.colorbar()
def GC_distribution_check_single_session(folder,
animal,
day,
gc_dist,
fast_samp='min',
show=False):
# check one session's GC value against whole gc_distribution
# TODO: dont forget to include SNRs for different neural sites
utils = os.path.join(folder, 'utils')
processed = os.path.join(folder, 'processed')
roi_type = get_roi_type(processed, animal, day)
pfname = encode_to_filename(utils, animal, day, hyperparams='granger')
GC_inds, ens_to_I = get_ens_to_indirect_GC(pfname, roi_type)
if ens_to_I.shape[0] == 0:
return np.nan
ksps = test_distribution(ens_to_I.ravel(),
gc_dist,
"{}_{}_GC".format(animal, day),
"GC_all_sessions",
"GC_ens-indirect" +
f"fastsamp_{fast_samp}" if fast_samp else "",
fast_samp=fast_samp,
show=show)
return ksps
def network_burst():
import h5py, os
animal, day = 'PT7', '181129'
folder = os.path.join("/Volumes/DATA_01/NL/layerproject", 'processed')
processed = folder
hf = h5py.File(encode_to_filename(processed, animal, day), 'r')
from utils_loading import encode_to_filename
hf = h5py.File(encode_to_filename(processed, animal, day), 'r')
dff = np.array(hf['dff'])
import matplotlib.pyplot as plt
import numpy as np
dff = np.array(hf['dff'])
hits = np.array(hf['hits'])
misses = np.array(hf['miss'])
SS = 0
MS = 1
plt.plot(np.mean(ens, axis=0));
plt.scatter(hits, np.full_like(hits, SS), s=MS, c='r');
plt.scatter(misses, np.full_like(misses, SS), s=MS, c='g');
plt.show()
ens = dff[np.array(hf['ens_neur']).astype(np.int)]
plt.plot(np.mean(ens, axis=0));
plt.scatter(hits, np.full_like(hits, SS), s=MS, c='r');
plt.scatter(misses, np.full_like(misses, SS), s=MS, c='g');
plt.show()
MS = -0.3
plt.plot(np.mean(ens, axis=0));
plt.scatter(hits, np.full_like(hits, SS), s=MS, c='r');
plt.scatter(misses, np.full_like(misses, SS), s=MS, c='g');
plt.show()
MS = 1
SS = -0.3
plt.plot(np.mean(ens, axis=0));
plt.scatter(hits, np.full_like(hits, SS), s=MS, c='r');
plt.scatter(misses, np.full_like(misses, SS), s=MS, c='g');
plt.show()
plt.plot(np.mean(ens, axis=0));
plt.scatter(hits, np.full_like(hits, SS), s=MS, c='r');
plt.scatter(misses, np.full_like(misses, SS), s=MS, c='g');
plt.show()
sig = np.mean(ens, axis=0)
def autocorr(x):
result = numpy.correlate(x, x, mode='full')
return result[result.size / 2:]
plt.plot(autocorr(sig))
import numpy
def autocorr(x):
result = numpy.correlate(x, x, mode='full')
return result[result.size / 2:]
plt.plot(autocorr(sig))
def autocorr(x):
result = np.correlate(x, x, mode='full')
return result[result.size // 2:]
plt.plot(autocorr(sig))
plt.show()
plt.plot(autocorr(sig))
plt.show()
firsthalf = sig[:, :9000]
sig.shape
firsthalf = sig[:9000]
lasthalf = sig[-9000:]
plt.plot(autocorr(firsthalf));
plt.plot(autocorr(lasthalf));
plt.legend();
plt.show()
plt.plot(autocorr(firsthalf));
plt.plot(autocorr(lasthalf));
plt.legend(['first', 'second']);
plt.show()
plt.plot(autocorr(firsthalf));
plt.plot(autocorr(lasthalf));
plt.legend(['first', 'second']);
plt.show()
plt.plot(np.diff(autocorr(firsthalf)));
plt.plot(np.diff(autocorr(lasthalf)));
plt.legend(['first', 'second']);
plt.show()
plt.plot(np.diff(autocorr(firsthalf)) + 2);
plt.plot(np.diff(autocorr(lasthalf)));
plt.legend(['first', 'second']);
plt.show()
plt.plot(np.diff(autocorr(firsthalf)) + 2);
plt.plot(np.diff(autocorr(lasthalf)));
plt.legend(['first', 'second']);
plt.show()
plt.plot(autocorr(firsthalf));
plt.plot(autocorr(lasthalf));
plt.legend(['first', 'second']);
plt.show()
plt.ylabel('IC');
plt.xlabel('lag');
plt.xticks(np.arange(0, 16), np.arange(0, 16))
plt.plot(np.arange(16), aics);
plt.plot(np.arange(16), mod.select_order(15).ics['bic']);
plt.ylabel('IC');
plt.xlabel('lag');
plt.legend(['aic', 'bic']);
plt.xticks(np.arange(0, 16), np.arange(0, 16))
from tests import *
processed = "/Users/albertqu/Documents/7.Research/BMI/analysis_data/processed/"
animal, day = "IT4", "181004"
hf = h5py.File(encode_to_filename(processed, animal, day), 'r')
nerden = np.array(hf['nerden'])
dff = np.array(hf['dff'])[nerden]
blen = hf.attrs['blen']
roi_type = get_roi_type(processed, animal, day)
ner_roi = roi_type[nerden]
ner_ens_sel = (ner_roi == 'E1') | (ner_roi == 'E2') | (ner_roi == 'E')
dff_e = dff[ner_ens_sel]
plt.xcorr(dff_e[3], dff[43])
from statsmodels.tsa.stattools import grangercausalitytests
res1 = grangercausalitytests(np.vstack([dff_e[3], dff[43]]).T, 4, verbose=False)
dff_e_shuffle = deconvolve_reconvolve(dff_e[3], shuffle=True)
dff_ind_shuffle = deconvolve_reconvolve(dff[43], shuffle=False)
def digitize_calcium(inputs, source, n):
"""Returns a pd.DataFrame with peak timing for calcium events
Params:
inputs: str, h5py.File, tuple, or np.ndarray
if str/h5py.File: string that represents the filename of hdf5 file
if tuple: (path, animal, day), that describes the file location
if np.ndarray: array C of calcium traces
out: str
Output path for saving the metrics in a hdf5 file
outfile: Animal_Day.csv
columns: neuron number
"""
if isinstance(inputs, np.ndarray):
S = inputs
animal, day = None, None
path = './'
savepath = os.path.join(path, 'sample_IBI_{}.csv')
else:
if isinstance(inputs, str):
opts = path_prefix_free(inputs, '/').split('_')
path = file_folder_path(inputs)
animal, day = opts[1], opts[2]
f = None
hfile = inputs
elif isinstance(inputs, tuple):
path, animal, day = inputs
f1 = os.path.join(path, animal,
"full_{}_{}__data.hdf5".format(animal, day))
f2 = encode_to_filename(path, animal, day)
if os.path.exists(f1):
hfile = f1
elif os.path.exists(f2):
hfile = f2
else:
raise FileNotFoundError("File {} or {} not found".format(
f1, f2))
f = None
elif isinstance(inputs, h5py.File):
opts = path_prefix_free(inputs.filename, '/').split('_')
path = file_folder_path(inputs.filename)
animal, day = opts[1], opts[2]
f = inputs
else:
raise RuntimeError("Input Format Unknown!")
savepath = os.path.join(path, '%s_%s_rawcwt_{}.csv' % (animal, day))
if os.path.exists(savepath):
return
if f is None:
f = h5py.File(hfile, 'r')
S = np.array(f[source])
f.close()
dgs = digitize_signal(S, n)
hyperparams = "n_{}".format(n)
savepath = savepath.format(hyperparams)
with open(savepath, 'w') as fh:
cwriter = csv.writer(fh, delimiter=',')
for i in range(S.shape[0]):
print(i)
cwriter.writerow(dgs[i])
return savepath
def digitize_calcium_by_animal(folder, animal, days, source, n):
for day in days:
if day.isnumeric():
hf = encode_to_filename(folder, animal, day)
digitize_calcium(hf, source, n)
def plot_peak_psth(folder,
animal,
day,
method,
window,
tlock=30,
eps=True,
t=True,
w=True):
# TODO: ADD CV TUNING
processed = os.path.join(folder, 'CaBMI_analysis/processed/')
psth = os.path.join(folder, 'bursting/plots/PSTH/')
windowplot = os.path.join(psth, 'window', animal, day)
trialplot = os.path.join(psth, 'trial', animal, day)
D_trial, D_window = get_peak_times_over_thres((processed, animal, day),
window,
method,
tlock=tlock)
# LABEL NEURON IN FRONT
with h5py.File(encode_to_filename(processed, animal, day), 'r') as f:
C = np.array(f['C'])
hits = np.array(f['array_t1'])
misses = np.array(f['array_miss'])
array_start = np.array(f['trial_start'])
array_end = np.array(f['trial_end'])
roi_types = get_roi_type(processed, animal, day)
hp = "psth_window_{}_theta_{}".format(window, decode_method_ibi(method)[1])
metrics = ('cv', 'cv_ub', 'serr_pc')
for i in range(C.shape[0]):
nstr = roi_types[i] + "/" + roi_types[i] + "_" + str(i)
ntfolder = os.path.join(trialplot, nstr)
nwfolder = os.path.join(windowplot, nstr)
fnamet = os.path.join(ntfolder, animal + "_" + day + "_" + hp)
fnamew = os.path.join(nwfolder, animal + "_" + day + "_" + hp)
if not os.path.exists(ntfolder):
os.makedirs(ntfolder)
if not os.path.exists((nwfolder)):
os.makedirs(nwfolder)
# TRIAL
if t and not os.path.exists(fnamet):
ibis_hit = [np.diff(D_trial[i][j]) for j in hits]
ibis_hit_mat = np.full((len(hits), len(max(ibis_hit, key=len))),
np.nan)
for j in range(len(hits)):
ibis_hit_mat[j, :len(ibis_hit[j])] = ibis_hit[j]
ibis_miss = [np.diff(D_trial[i][j]) for j in misses]
ibis_miss_mat = np.full(
(len(misses), len(max(ibis_miss, key=len))), np.nan)
for j in range(len(misses)):
ibis_miss_mat[j, :len(ibis_miss[j])] = ibis_miss[j]
fig = plt.figure(figsize=(20, 10))
hitsx = np.concatenate(
[np.array(D_trial[i][j]) - array_end[j] for j in hits])
hitsy = np.concatenate(
[np.full(len(D_trial[i][j]), j + 1) for j in hits])
missx = np.concatenate(
[np.array(D_trial[i][j]) - array_end[j] for j in misses])
missy = np.concatenate(
[np.full(len(D_trial[i][j]), j + 1) for j in misses])
gs = gridspec.GridSpec(3, 1, height_ratios=[5, 1, 1])
ax0 = plt.subplot(gs[0])
ax1 = plt.subplot(gs[1])
ax2 = plt.subplot(gs[2])
plt.subplots_adjust(hspace=0.5)
ax0.plot(hitsx, hitsy, 'b.', markersize=3)
ax0.plot(missx, missy, 'k.', markersize=3)
ax0.plot(array_start - array_end,
np.arange(1,
len(array_start) + 1),
'r.',
markersize=3)
ax0.legend(['hits', 'miss', 'trial start'])
ax0.axvline(0, color='r')
ax0.set_title("PSTH trial")
ax0.set_xlabel('Time(fr)')
ax0.set_ylabel('Trial Number')
blues, reds = sns.color_palette("Blues",
3), sns.color_palette('Reds', 3)
for j, m in enumerate(metrics):
t1 = IBI_cv_matrix(ibis_hit_mat, m)
t2 = IBI_cv_matrix(ibis_miss_mat, m)
ax1.plot(hits, (t1 - np.nanmin(t1)) /
(np.nanmax(t1) - np.nanmin(t1)),
c=blues[j],
label='hit ' + m)
ax1.plot(misses, (t2 - np.nanmin(t2)) /
(np.nanmax(t2) - np.nanmin(t2)),
c=reds[j],
label='miss ' + m)
ax1.set_title("HM cv evolution")
ax1.set_ylabel('minmax scale')
ax1.set_xlabel("Trial#")
ax2.plot(hits, [np.nanmean(h) for h in ibis_hit])
ax2.plot(misses, [np.nanmean(mi) for mi in ibis_miss])
ax2.legend(['hit', 'miss'])
ax2.set_xlabel("Trial#")
ax2.set_ylabel("No. of Frames")
#TODO: MAYBE ADD A PSTH here
fig.savefig(fnamet + '.png')
if eps:
fig.savefig(fnamet + ".eps")
plt.close('all')
# WINDOW
if w and not os.path.exists(fnamew):
fig = plt.figure(figsize=(20, 10))
wlen = len(D_window[i])
slidex = np.concatenate(
[np.array(D_window[i][j]) - window * j for j in range(wlen)])
slidey = np.concatenate(
[np.full(len(D_window[i][j]), j + 1) for j in range(wlen)])
ibis_slide = [np.diff(D_window[i][j]) for j in range(wlen)]
ibis_slide_mat = np.full((wlen, len(max(ibis_slide, key=len))),
np.nan)
for j in range(wlen):
ibis_slide_mat[j, :len(ibis_slide[j])] = ibis_slide[j]
gs = gridspec.GridSpec(3, 1, height_ratios=[5, 1, 1])
ax0 = plt.subplot(gs[0])
ax1 = plt.subplot(gs[1])
ax2 = plt.subplot(gs[2])
plt.subplots_adjust(hspace=0.5)
ax0.plot(slidex, slidey, 'k.', markersize=3)
ax0.axvline(0, color='r')
ax0.set_title("PSTH window")
ax0.set_xlabel('Time(fr)')
ax0.set_ylabel('Slide')
for m in metrics:
temp = IBI_cv_matrix(ibis_slide_mat, m)
ax1.plot(np.arange(wlen), (temp - np.nanmin(temp)) /
(np.nanmax(temp) - np.nanmin(temp)))
ax1.legend(metrics)
ax1.set_title("CV evolution")
ax1.set_ylabel('minmax scale')
ax1.set_xlabel("Window")
print(i, len(ibis_slide))
try:
ax2.hist(ibis_slide,
density=True,
color=sns.color_palette("Blues", wlen))
except:
print(ibis_slide)
ax2.legend(np.arange(wlen))
ax2.set_title("IBI dist evolution")
fig.savefig(fnamew + '.png')
if eps:
fig.savefig(fnamew + ".eps")
plt.close("all")
def dff_sanity_check_single_session(rawbase, processed, animal, day, out=None, PROBELEN=1000,
number_planes_total=6, mproc=False):
rawpath = os.path.join(rawbase, animal, day)
end = 10
onlinef = None
for f in os.listdir(rawpath):
if f.find('bmi_IntegrationRois') != -1:
tend = int(f[-5])
if tend < end:
end = tend
onlinef = f
if onlinef is None:
raise FileNotFoundError('bmi_IntegrationRois')
online_data = pd.read_csv(os.path.join(rawpath, onlinef))
f1 = os.path.join(processed, animal, "full_{}_{}__data.hdf5".format(animal, day))
f2 = encode_to_filename(processed, animal, day)
if os.path.exists(f1):
hfname = f1
elif os.path.exists(f2):
hfname = f2
else:
raise FileNotFoundError("File {} or {} not found".format(f1, f2))
with h5py.File(hfname, 'r') as hf:
dff = np.array(hf['dff'])
C = np.array(hf['C'])
blen = hf.attrs['blen']
ens_neur = np.array(hf['ens_neur'])
dff[np.isnan(dff)] = 0
dff_ens = dff[ens_neur]
C_ens = C[ens_neur]
units = len(ens_neur)
N = 2 * units
def helper(vars):
R = np.corrcoef(vars)
corrs_pair = np.diagonal(R, units)
chance_corr = (np.nansum(R) / 2 - units - np.nansum(corrs_pair)) * 2 / (N ** 2 - 2 * N)
return corrs_pair, chance_corr
corrs_pair1, chance1 = helper(np.vstack([dff_ens, C_ens]))
frames = online_data['frameNumber'].values // number_planes_total + blen
online = online_data.iloc[:, 2:2 + units].values.T
online[np.isnan(online)] = 0
slice_stack = np.vstack([dff_ens[:, frames], online])
corrs_pair2, chance2 = helper(slice_stack)
b = [np.nan] * 4
corrs_pair3, chance3 = helper(np.vstack([C[:, frames], online]))
if out is not None:
CAIMANONLY = False
OFFSET = 0
fig, axes = plt.subplots(2, 2, figsize=(20, 15))
axflat = axes.ravel()
for i, ens in enumerate(ens_neur):
if CAIMANONLY:
axflat[i].plot(zscore(dff[ens]))
axflat[i].plot(zscore(C[ens]) + OFFSET)
axflat[i].legend(['CaImAnDFF', 'C'])
axflat[i].set_title('Ens #{}'.format(ens))
else:
# s = online_data.iloc[:, 2+i].values
s = online[i]
nmean = np.nanmean(s)
auxonline = (s - nmean) / nmean
onlinedff = auxonline
onlineraw = s
# TODO: get back to Nuria for the sample analysis plots in harddrive of the
# ensemble vs C plots
axflat[i].plot(zscore(dff[ens, frames[-PROBELEN:]]))
axflat[i].plot(zscore(C[ens, frames[-PROBELEN:]]) + OFFSET * 1)
# axflat[i].plot(zscore(onlinedff[-PROBELEN:]) + OFFSET * 2)
axflat[i].plot(zscore(onlineraw[-PROBELEN:]) + OFFSET * 2)
# axflat[i].legend(['CaImAnDFF', 'C', 'greedyDFF(f0=mean)', 'online raw'])
axflat[i].legend(['CaImAnDFF', 'C', 'online raw'])
axflat[i].set_title('Ens #{}'.format(ens))
fig.suptitle("CaImAn DFF Sanity Check {} {}{}".format(animal, day,
" With Offset {}".format(
OFFSET) if OFFSET else ""))
basename = "dff_check_{}_{}{}{}".format(animal, day,
"" if CAIMANONLY else "_with_raw_online",
"_offset_{}".format(OFFSET) if OFFSET else "")
tpath = os.path.join(out, "OFFSET{}".format(OFFSET))
if not os.path.exists(tpath):
os.makedirs(tpath)
outname = os.path.join(out, "OFFSET{}".format(OFFSET), basename)
fig.savefig(outname + '.png')
fig.savefig(outname + '.eps')
if not mproc:
plt.show()
results = [animal, day, chance1] + b + [chance2] + b + [chance3] +b
for i in range(units):
results[i + 3] = corrs_pair1[i]
results[i + 8] = corrs_pair2[i]
results[i + 13] = corrs_pair3[i]
return results
def ens_to_ind_GC_double_reconv_shuffle_test_single_session(
folder,
animal,
day,
test_reconv=True,
thres=0.05,
snr_thres=0,
lagmode=2):
""" Calculates granger causality from ensemble to indirect neurons
:param ens_dff: E * T, E: number ensemble neurons
:param ind_dff: I * T, I: number of indirect neurons
:return:
"""
# get ens_dff, get_ind_dff
utils = os.path.join(folder, 'utils')
processed = os.path.join(folder, 'processed')
with h5py.File(encode_to_filename(processed, animal, day), 'r') as hfile:
roi_type = get_roi_type(hfile, animal, day)
blen = hfile.attrs['blen']
e_sel = np.isin(roi_type, ['E1', 'E2', 'E'])
inds = np.arange(len(roi_type))
ir_sel = roi_type == 'IR'
ig_sel = roi_type == 'IG'
dff = np.array(hfile['dff'])[:, :blen]
dff_e = dff[e_sel, :]
dff_ir = dff[ir_sel, :]
dff_ig = dff[ig_sel, :]
dff_inds = np.vstack([dff_ir, dff_ig])
GC_hf_inds = np.concatenate([inds[e_sel], inds[ir_sel], inds[ig_sel]])
if dff_e.shape[0] > 0:
# get GC ens_dff to ind_dff
pfname = encode_to_filename(utils, animal, day, hyperparams='granger')
GC_inds, ens_to_I = get_ens_to_indirect_GC(pfname,
roi_type,
thres=0.05)
shuf_file_found = False
save_shufmat = {}
try:
shuf_file = encode_to_filename(utils,
animal,
day,
hyperparams='reconv_shuffle')
shuf_file_found = True
shufmat = loadmat(shuf_file)
dff_e_rshuffle = shufmat['dff_e_rshuffle']
dff_ind_rshuffle = shufmat['dff_ind_rshuffle']
except FileNotFoundError:
# TODO: take care of rows of nans
# reconvolve shuffle dff_e and dff_ind
dff_e_rshuffle = np.vstack([
deconvolve_reconvolve(dff_e[i], shuffle=True)
for i in range(dff_e.shape[0])
])
dff_ind_rshuffle = np.vstack([
deconvolve_reconvolve(dff_inds[i], shuffle=True)
for i in range(dff_inds.shape[0])
])
save_shufmat['dff_e_rshuffle'] = dff_e_rshuffle
save_shufmat['dff_ind_rshuffle'] = dff_ind_rshuffle
save_shufmat['ens_to_I'] = ens_to_I
save_shufmat['GC_inds'] = GC_inds
dff_all_rshuffle = np.vstack([dff_e_rshuffle, dff_ind_rshuffle])
nansel_rshuffle = np.any(np.isnan(dff_all_rshuffle), axis=1)
# calculate granger causality for the reconvolved data
if isinstance(lagmode, str):
try:
lag = granger_select_order(dff_all_rshuffle[~nansel_rshuffle],
maxlag=5,
ic='bic')
except:
lag = 2
else:
lag = lagmode
gcs_val1, p_vals1 = statsmodel_granger_asymmetric(
dff_e_rshuffle, dff_ind_rshuffle, lag, False)
p_vals1 = p_vals1['ssr_chi2test']
gcs_val1[p_vals1 > thres] = 0
if not shuf_file_found:
save_shufmat['gcs_rshuffle'] = gcs_val1
if lagmode == 'max':
gcs_val1 = np.max(gcs_val1, axis=2)
else:
# auto or number
gcs_val1 = gcs_val1[:, :, -1]
assert gcs_val1.shape == ens_to_I.shape
assert np.allclose(GC_hf_inds, GC_inds)
# reconv comparison
if test_reconv:
# reconvolve dff_e and dff_ind
if shuf_file_found:
dff_e_reconv = shufmat['dff_e_reconv']
dff_ind_reconv = shufmat['dff_ind_reconv']
else:
dff_e_reconv = np.vstack([
deconvolve_reconvolve(dff_e[i])
for i in range(dff_e.shape[0])
])
dff_ind_reconv = np.vstack([
deconvolve_reconvolve(dff_inds[i])
for i in range(dff_inds.shape[0])
])
save_shufmat['dff_e_reconv'] = dff_e_reconv
save_shufmat['dff_ind_reconv'] = dff_ind_reconv
dff_all_reconv = np.vstack([dff_e_reconv, dff_ind_reconv])
nansel_reconv = np.any(np.isnan(dff_all_reconv), axis=1)
if isinstance(lagmode, str):
try:
lag = granger_select_order(dff_all_reconv[~nansel_reconv],
maxlag=5,
ic='bic')
except:
lag = 2
else:
lag = lagmode
gcs_val0, p_vals0 = statsmodel_granger_asymmetric(
dff_e_reconv, dff_ind_reconv, lag, False)
p_vals0 = p_vals0['ssr_chi2test']
gcs_val0[p_vals0 > thres] = 0
if not shuf_file_found:
save_shufmat['gcs_reconv'] = gcs_val0
if lagmode == 'max':
gcs_val0 = np.max(gcs_val0, axis=2)
else:
# auto or number
gcs_val0 = gcs_val0[:, :, -1]
assert gcs_val0.shape == ens_to_I.shape
assert np.allclose(GC_hf_inds, GC_inds)
snr_e = np.array([
caiman_SNR(None, dff_e[i], 'fast') for i in range(len(dff_e))
])
snr_inds = np.array([
caiman_SNR(None, dff_inds[i], 'fast')
for i in range(len(dff_inds))
])
valid_sel_e = snr_e > snr_thres
valid_sel_inds = snr_inds > snr_thres
ens_to_I_valid = ens_to_I[valid_sel_e, :][:, valid_sel_inds]
gcs_val0_valid = gcs_val0[valid_sel_e, :][:, valid_sel_inds]
gcs_val1_valid = gcs_val1[valid_sel_e, :][:, valid_sel_inds]
results = {
'dff_e_reconv': dff_e_reconv,
'dff_ind_reconv': dff_ind_reconv,
'dff_e_rshuffle': dff_e_rshuffle,
'dff_ind_rshuffle': dff_ind_reconv,
'ens_to_I': ens_to_I,
'GC_inds': GC_inds,
'gcs_rshuffle': gcs_val1,
'gcs_reconv': gcs_val0
}
savemat(
os.path.join(utils, 'FC/statsmodel/', animal, day,
f'{animal}_{day}_reconv_shuffle_gc.mat'), results)
fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(20, 10))
plt.subplots_adjust(hspace=0.4)
visualize_gc_pairs(ens_to_I_valid,
gcs_val0_valid,
"GC",
"reconvGC",
axes[0],
diff_label='d(raw,reconv)',
verbose=True,
firstbin=False,
kde=False)
visualize_gc_pairs(gcs_val0_valid,
gcs_val1_valid,
"reconvGC",
"rshufGC",
axes[1],
diff_label='normalized',
verbose=True,
firstbin=False,
kde=False)
visualize_gc_pairs(ens_to_I_valid,
gcs_val1_valid,
"GC",
"rshufGC",
axes[2],
diff_label='normalized',
verbose=True,
firstbin=False,
kde=False)
else:
fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(20, 10))
visualize_gc_pairs(ens_to_I,
gcs_val1,
"GC",
"rshufGC",
axes,
diff_label='normalized',
verbose=True,
firstbin=False,
kde=False)
if not shuf_file_found:
savemat(
encode_to_filename(utils,
animal,
day,
hyperparams='reconv_shuffle',
err=False), save_shufmat)
else:
raise RuntimeError(f"No ensemble neuron in {animal} {day}")