def statsmodel_solution(animal, day, maxlag=5): import h5py import numpy as np from utils_loading import encode_to_filename import matplotlib.pyplot as plt from statsmodels.tsa.stattools import grangercausalitytests folder = "/Volumes/DATA_01/NL/layerproject/processed/" hf = h5py.File(encode_to_filename(folder, animal, day), 'r') dff = np.array(hf['dff']) NEUR = 'ens' # Critical neuron pair vs general neuron gc if NEUR == 'ens': rois = dff[hf['ens_neur']] elif NEUR == 'neur': rois = dff[hf['nerden']] else: rois = dff[NEUR] # rois: N * T gcs_val = np.zeros((rois.shape[0], rois.shape[0], maxlag)) tests = ['ssr_ftest', 'ssr_chi2test', 'lrtest','params_ftest'] p_vals = {t:np.zeros((rois.shape[0], rois.shape[0], maxlag)) for t in tests} for i in range(rois.shape[0]): for j in range(rois.shape[0]): res = grangercausalitytests(rois[[j, i]].T, maxlag) for k in res: test, reg = res[k] ssrEig = reg[0].ssr ssrBeid = reg[1].ssr gcs_val[i, j, k-1] = np.log(ssrEig / ssrBeid) for t in tests: p_vals[t][i, j, k-1] = test[t][1] #TODO: USE LOG stats of two ssrs return gcs_val, p_vals
def digitize_calcium_all(folder, groups, source, ns, nproc=1): # TODO: ADD OPTION TO PASS IN A LIST OF METHODS FOR COMPARING THE PLOTS! """Calculates Peak Timing and Stores them in csvs for all animal sessions in groups located in folder.""" processed = os.path.join(folder, 'CaBMI_analysis/processed') logfolder = os.path.join(processed, 'log') if not os.path.exists(logfolder): os.makedirs(logfolder) all_files = parse_group_dict(processed, groups, 'all') print(all_files) for n in ns: if nproc == 0: nproc = mp.cpu_count() if nproc == 1: for animal in all_files: for day in (all_files[animal]): print(animal, day) hf = encode_to_filename(processed, animal, day) digitize_calcium(hf, source, n) else: p = mp.Pool(nproc) p.starmap_async(digitize_calcium_by_animal, [(processed, animal, all_files[animal], source, n) for animal in all_files]) with open("dCalcium_n_{}.txt".format(n)) as f: f.write("done")
def get_roi_type(processed, animal, day): # TODO: fix problems when ens becomes None rois = None if isinstance(processed, str): hfile = h5py.File(encode_to_filename(processed, animal, day), 'r') else: hfile = processed N = hfile['C'].shape[0] rois = np.full(N, "D", dtype="U2") nerden = np.array(hfile['nerden']) redlabel = np.array(hfile['redlabel']) ens_neur = np.array(hfile['ens_neur']) ens_neur = ens_neur[~np.isnan(ens_neur)].astype(np.int) e2_neur = None if 'e2_neur' in hfile: temp = np.array(hfile['e2_neur']) if ~np.any(np.isnan(temp)): e2_neur = temp.astype(np.int) if isinstance(processed, str): hfile.close() rois[nerden & ~redlabel] = 'IG' rois[nerden & redlabel] = 'IR' if e2_neur is not None: rois[ens_neur] = 'E1' rois[e2_neur] = 'E2' else: rois[ens_neur] = 'E' return rois
def deconvolve_reconvolve_single_session_test(processed, animal, day, randN=None, savePlot=None, showPlot=True, **kwargs): with h5py.File(encode_to_filename(processed, animal, day), 'r') as hfile: roi_type = get_roi_type(hfile, animal, day) e_sel = np.isin(roi_type, ['E1', 'E2', 'E']) ind_sel = np.isin(roi_type, ['IR', 'IG']) nerden = np.array(hfile['nerden']) dff = np.array(hfile['dff']) dff_e = dff[e_sel, :] dff_ind = dff[ind_sel, :] dff_all = np.vstack([dff_e, dff_ind]) if randN is not None: rinds = np.arange(dff_all.shape[0]) np.random.shuffle(rinds) dff_all = dff_all[rinds[:randN]] reconvs = np.full_like(dff_all, np.nan) all_sn = np.full(dff_all.shape[0], np.nan) recleans = np.full_like(dff_all, np.nan) valid_selector = np.full(dff_all.shape[0], 1, dtype=bool) for i in range(dff_all.shape[0]): print(i) try: c, c1, sp, sn, reconv, reconv_clean = deconvolve_reconvolve_test( dff_all[i], tag=str(i), show=False, save=None, **kwargs) reconvs[i] = reconv all_sn[i] = sn recleans[i] = reconv_clean except: valid_selector[i] = False TAG = f'{animal}_{day}_dff_reconvolve' for nthres in [0, 0.5, 1]: corrs_clean = noise_free_corr(dff_all[valid_selector], recleans[valid_selector], all_sn[valid_selector], noise_thres=nthres, tag=TAG, save=savePlot, show=showPlot) # corrs = noise_free_corr(dff_all[valid_selector], reconvs[valid_selector], all_sn[valid_selector], # None, noise_thres=nthres, tag=TAG, save=savePlot, # show=showPlot) ksps = test_distribution(dff_all[valid_selector], reconvs[valid_selector], tag1='dff', tag2='reconvolve', alltag=TAG, save=savePlot, show=showPlot) return corrs_clean, ksps
def raw_activity_tuning_single_session(folder, animal, day, i, window=3000, itype='dff', metric='raw', zcap=None): hf = encode_to_filename(processed, animal, day) if not os.path.exists(hf): print("Not found:, ", hf) with h5py.File(hf, 'r') as fp: S = np.array(fp[itype]) # TODO: maybe include trial activity tuning #array_hit, array_miss = np.array(fp['array_t1']), np.array(fp['array_miss']) ens_neur = np.array(fp['ens_neur']) e2_neur = ens_neur[fp['e2_neur']] if 'e2_neur' in fp else None redlabel, nerden = np.array(fp['redlabel']), np.array(fp['nerden']) rois = get_roi_type(fp, animal, day) if metric == 'mean' and zcap is not None: zscoreS = zscore(S, axis=1) if zcap != -1: zscoreS = np.minimum(zscoreS, np.full_like(S, zcap)) else: zscoreS = S N = S.shape[0] nsessions = S.shape[1] // window remainder = S.shape[1] - nsessions * window Sfirst = (zscoreS[:, :nsessions * window]).reshape( (S.shape[0], nsessions, window), order='C') avg_S = np.nanmean(Sfirst, axis=2) if remainder > 0: Sremain = zscoreS[:, nsessions * window:] avg_S = np.concatenate( (avg_S, np.nanmean(Sremain, keepdims=True, axis=1)), axis=1) # N, sw, st = probeW.shape[0], probeW.shape[1], probeT.shape[1] # ROI_type sw = nsessions + (1 if remainder else 0) # DF Window results = [ np.tile(np.arange(sw), N), np.repeat(rois, sw), np.repeat(np.arange(N), sw), avg_S.ravel(order='C'), np.full(N * sw, animal[:2]), np.full(N * sw, animal), np.full(N * sw, day), np.full(N * sw, i + 1) ] print(animal, day, 'done') return results
def __init__(self, folder, animal, day, sec_var='', lag=2, method='te-extended', out=None): """ method: str xc, by finding the peak in the cross-correlogram between the two time series mi, by finding the lag with the largest Mutual Information gc, by computing the Granger Causality, based on: C.W.J. Granger, Investigating Causal Relations by Econometric Models and Cross-Spectral Methods , Econometrica, 1969 te-extended, by computing GTE as defined above. TE and GTE without binning: te-binless-Leonenko, based on: L.F. Kozachenko and N.N. Leonenko, 1987 te-binless-Kraskov, based on: A. Kraskov et al., 2004 te-binless-Frenzel, based on: S. Frenzel and B. Pompe, 2007 te-symbolic (experimental) based on: M. Staniek and K. Lehnertz, 2008. out: str path of the root output directory, if left None, default to {folder}/utils/FC/{method}/ """ if out is None: out = os.path.join(folder, 'utils/FC/') self.out_path = os.path.join(out, f'te-package_' + method, animal, day, '') if not os.path.exists(self.out_path): os.makedirs(self.out_path) self.folder = folder self.animal = animal self.day = day self.parameters = { "AutoConditioningLevelQ": True, 'AutoBinNumberQ': True, 'SourceMarkovOrder': lag, 'TargetMarkovOrder': lag, 'StartSampleIndex': 2 } # update conditioning level for gte self.exp_file = h5py.File( encode_to_filename(os.path.join(folder, 'processed'), animal, day), 'r') self.blen = self.exp_file.attrs['blen'] self.method = method
def get_general_GC_distribution(folder, resamp=0.1, concat=True, save=True): """ Plots distplot for the entire ens-indirect GC distribution Input: folder: root folder """ fc = os.path.join(folder, "utils/FC") rsampTag = f"_resamp{resamp}" if resamp else "" alldistf = os.path.join(fc, f"gc_ens_to_I_order2_alldist{rsampTag}.npy") if os.path.exists(alldistf): assert concat, "fast load only supports concatenated format" return np.load(alldistf) utils = os.path.join(folder, 'utils') processed = os.path.join(folder, 'processed') alles = [] for animal in get_all_animals(processed): animal_path = os.path.join(processed, animal) for day in get_animal_days(animal_path): print(animal, day) roi_type = get_roi_type(processed, animal, day) pfname = encode_to_filename(utils, animal, day, hyperparams='granger') GC_inds, ens_to_I = get_ens_to_indirect_GC(pfname, roi_type) ens_to_I_rav = ens_to_I.ravel() if resamp is not None: resampsize = int(resamp * len(ens_to_I_rav)) # TODO: maybe fix seeds? rand_inds = sample_without_replacement( n_population=len(ens_to_I_rav), n_samples=resampsize) ens_to_I_rav = ens_to_I_rav[rand_inds] alles.append(ens_to_I_rav) concats = np.concatenate(alles) if save: np.save(alldistf, concats) if concat: return concats else: return np.vstack(alles)
def calcium_to_peak_times_all(folder, groups, low=1, high=20): # TODO: ADD OPTION TO PASS IN A LIST OF METHODS FOR COMPARING THE PLOTS! """Calculates Peak Timing and Stores them in csvs for all animal sessions in groups located in folder.""" processed = os.path.join(folder, 'CaBMI_analysis/processed') if groups == '*': all_files = get_PTIT_over_days(processed) else: all_files = { g: parse_group_dict(processed, groups[g], g) for g in groups.keys() } print(all_files) for group in all_files: group_dict = all_files[group] for animal in group_dict: for day in (group_dict[animal]): print(animal, day) hf = encode_to_filename(processed, animal, day) calcium_to_peak_times(hf, low, high)
def nitime_solution(animal, day, cutoff=True): # TODO: add support for custom freq bands import h5py import nitime import numpy as np import nitime.analysis as nta from utils_loading import encode_to_filename from nitime.viz import drawmatrix_channels import matplotlib.pyplot as plt folder = "/Volumes/DATA_01/NL/layerproject/processed/" hf = h5py.File(encode_to_filename(folder, animal, day), 'r') dff = np.array(hf['dff']) NEUR = 'ens' # Critical neuron pair vs general neuron gc if NEUR == 'ens': rois = dff[hf['ens_neur']] elif NEUR == 'neur': rois = dff[hf['nerden']] else: rois = dff[NEUR] rois_ts = nitime.TimeSeries(rois, sampling_interval=1 / hf.attrs['fr']) G = nta.GrangerAnalyzer(rois_ts) if cutoff: sel = np.where(G.frequencies < hf.attrs['fr'])[0] caus_xy = G.causality_xy[:, :, sel] caus_yx = G.causality_yx[:, :, sel] caus_sim = G.simultaneous_causality[:, :, sel] else: caus_xy = G.causality_xy caus_yx = G.causality_yx caus_sim = G.simultaneous_causality g1 = np.mean(caus_xy, -1) g2 = np.mean(caus_yx, -1) g3 = np.mean(caus_sim, -1) g4 = g1-g2 fig03 = drawmatrix_channels(g1, ['E11', 'E12', 'E21', 'E22'], size=[10., 10.], color_anchor = 0) plt.colorbar()
def GC_distribution_check_single_session(folder, animal, day, gc_dist, fast_samp='min', show=False): # check one session's GC value against whole gc_distribution # TODO: dont forget to include SNRs for different neural sites utils = os.path.join(folder, 'utils') processed = os.path.join(folder, 'processed') roi_type = get_roi_type(processed, animal, day) pfname = encode_to_filename(utils, animal, day, hyperparams='granger') GC_inds, ens_to_I = get_ens_to_indirect_GC(pfname, roi_type) if ens_to_I.shape[0] == 0: return np.nan ksps = test_distribution(ens_to_I.ravel(), gc_dist, "{}_{}_GC".format(animal, day), "GC_all_sessions", "GC_ens-indirect" + f"fastsamp_{fast_samp}" if fast_samp else "", fast_samp=fast_samp, show=show) return ksps
def network_burst(): import h5py, os animal, day = 'PT7', '181129' folder = os.path.join("/Volumes/DATA_01/NL/layerproject", 'processed') processed = folder hf = h5py.File(encode_to_filename(processed, animal, day), 'r') from utils_loading import encode_to_filename hf = h5py.File(encode_to_filename(processed, animal, day), 'r') dff = np.array(hf['dff']) import matplotlib.pyplot as plt import numpy as np dff = np.array(hf['dff']) hits = np.array(hf['hits']) misses = np.array(hf['miss']) SS = 0 MS = 1 plt.plot(np.mean(ens, axis=0)); plt.scatter(hits, np.full_like(hits, SS), s=MS, c='r'); plt.scatter(misses, np.full_like(misses, SS), s=MS, c='g'); plt.show() ens = dff[np.array(hf['ens_neur']).astype(np.int)] plt.plot(np.mean(ens, axis=0)); plt.scatter(hits, np.full_like(hits, SS), s=MS, c='r'); plt.scatter(misses, np.full_like(misses, SS), s=MS, c='g'); plt.show() MS = -0.3 plt.plot(np.mean(ens, axis=0)); plt.scatter(hits, np.full_like(hits, SS), s=MS, c='r'); plt.scatter(misses, np.full_like(misses, SS), s=MS, c='g'); plt.show() MS = 1 SS = -0.3 plt.plot(np.mean(ens, axis=0)); plt.scatter(hits, np.full_like(hits, SS), s=MS, c='r'); plt.scatter(misses, np.full_like(misses, SS), s=MS, c='g'); plt.show() plt.plot(np.mean(ens, axis=0)); plt.scatter(hits, np.full_like(hits, SS), s=MS, c='r'); plt.scatter(misses, np.full_like(misses, SS), s=MS, c='g'); plt.show() sig = np.mean(ens, axis=0) def autocorr(x): result = numpy.correlate(x, x, mode='full') return result[result.size / 2:] plt.plot(autocorr(sig)) import numpy def autocorr(x): result = numpy.correlate(x, x, mode='full') return result[result.size / 2:] plt.plot(autocorr(sig)) def autocorr(x): result = np.correlate(x, x, mode='full') return result[result.size // 2:] plt.plot(autocorr(sig)) plt.show() plt.plot(autocorr(sig)) plt.show() firsthalf = sig[:, :9000] sig.shape firsthalf = sig[:9000] lasthalf = sig[-9000:] plt.plot(autocorr(firsthalf)); plt.plot(autocorr(lasthalf)); plt.legend(); plt.show() plt.plot(autocorr(firsthalf)); plt.plot(autocorr(lasthalf)); plt.legend(['first', 'second']); plt.show() plt.plot(autocorr(firsthalf)); plt.plot(autocorr(lasthalf)); plt.legend(['first', 'second']); plt.show() plt.plot(np.diff(autocorr(firsthalf))); plt.plot(np.diff(autocorr(lasthalf))); plt.legend(['first', 'second']); plt.show() plt.plot(np.diff(autocorr(firsthalf)) + 2); plt.plot(np.diff(autocorr(lasthalf))); plt.legend(['first', 'second']); plt.show() plt.plot(np.diff(autocorr(firsthalf)) + 2); plt.plot(np.diff(autocorr(lasthalf))); plt.legend(['first', 'second']); plt.show() plt.plot(autocorr(firsthalf)); plt.plot(autocorr(lasthalf)); plt.legend(['first', 'second']); plt.show()
plt.ylabel('IC'); plt.xlabel('lag'); plt.xticks(np.arange(0, 16), np.arange(0, 16)) plt.plot(np.arange(16), aics); plt.plot(np.arange(16), mod.select_order(15).ics['bic']); plt.ylabel('IC'); plt.xlabel('lag'); plt.legend(['aic', 'bic']); plt.xticks(np.arange(0, 16), np.arange(0, 16)) from tests import * processed = "/Users/albertqu/Documents/7.Research/BMI/analysis_data/processed/" animal, day = "IT4", "181004" hf = h5py.File(encode_to_filename(processed, animal, day), 'r') nerden = np.array(hf['nerden']) dff = np.array(hf['dff'])[nerden] blen = hf.attrs['blen'] roi_type = get_roi_type(processed, animal, day) ner_roi = roi_type[nerden] ner_ens_sel = (ner_roi == 'E1') | (ner_roi == 'E2') | (ner_roi == 'E') dff_e = dff[ner_ens_sel] plt.xcorr(dff_e[3], dff[43]) from statsmodels.tsa.stattools import grangercausalitytests res1 = grangercausalitytests(np.vstack([dff_e[3], dff[43]]).T, 4, verbose=False) dff_e_shuffle = deconvolve_reconvolve(dff_e[3], shuffle=True) dff_ind_shuffle = deconvolve_reconvolve(dff[43], shuffle=False)
def digitize_calcium(inputs, source, n): """Returns a pd.DataFrame with peak timing for calcium events Params: inputs: str, h5py.File, tuple, or np.ndarray if str/h5py.File: string that represents the filename of hdf5 file if tuple: (path, animal, day), that describes the file location if np.ndarray: array C of calcium traces out: str Output path for saving the metrics in a hdf5 file outfile: Animal_Day.csv columns: neuron number """ if isinstance(inputs, np.ndarray): S = inputs animal, day = None, None path = './' savepath = os.path.join(path, 'sample_IBI_{}.csv') else: if isinstance(inputs, str): opts = path_prefix_free(inputs, '/').split('_') path = file_folder_path(inputs) animal, day = opts[1], opts[2] f = None hfile = inputs elif isinstance(inputs, tuple): path, animal, day = inputs f1 = os.path.join(path, animal, "full_{}_{}__data.hdf5".format(animal, day)) f2 = encode_to_filename(path, animal, day) if os.path.exists(f1): hfile = f1 elif os.path.exists(f2): hfile = f2 else: raise FileNotFoundError("File {} or {} not found".format( f1, f2)) f = None elif isinstance(inputs, h5py.File): opts = path_prefix_free(inputs.filename, '/').split('_') path = file_folder_path(inputs.filename) animal, day = opts[1], opts[2] f = inputs else: raise RuntimeError("Input Format Unknown!") savepath = os.path.join(path, '%s_%s_rawcwt_{}.csv' % (animal, day)) if os.path.exists(savepath): return if f is None: f = h5py.File(hfile, 'r') S = np.array(f[source]) f.close() dgs = digitize_signal(S, n) hyperparams = "n_{}".format(n) savepath = savepath.format(hyperparams) with open(savepath, 'w') as fh: cwriter = csv.writer(fh, delimiter=',') for i in range(S.shape[0]): print(i) cwriter.writerow(dgs[i]) return savepath
def digitize_calcium_by_animal(folder, animal, days, source, n): for day in days: if day.isnumeric(): hf = encode_to_filename(folder, animal, day) digitize_calcium(hf, source, n)
def plot_peak_psth(folder, animal, day, method, window, tlock=30, eps=True, t=True, w=True): # TODO: ADD CV TUNING processed = os.path.join(folder, 'CaBMI_analysis/processed/') psth = os.path.join(folder, 'bursting/plots/PSTH/') windowplot = os.path.join(psth, 'window', animal, day) trialplot = os.path.join(psth, 'trial', animal, day) D_trial, D_window = get_peak_times_over_thres((processed, animal, day), window, method, tlock=tlock) # LABEL NEURON IN FRONT with h5py.File(encode_to_filename(processed, animal, day), 'r') as f: C = np.array(f['C']) hits = np.array(f['array_t1']) misses = np.array(f['array_miss']) array_start = np.array(f['trial_start']) array_end = np.array(f['trial_end']) roi_types = get_roi_type(processed, animal, day) hp = "psth_window_{}_theta_{}".format(window, decode_method_ibi(method)[1]) metrics = ('cv', 'cv_ub', 'serr_pc') for i in range(C.shape[0]): nstr = roi_types[i] + "/" + roi_types[i] + "_" + str(i) ntfolder = os.path.join(trialplot, nstr) nwfolder = os.path.join(windowplot, nstr) fnamet = os.path.join(ntfolder, animal + "_" + day + "_" + hp) fnamew = os.path.join(nwfolder, animal + "_" + day + "_" + hp) if not os.path.exists(ntfolder): os.makedirs(ntfolder) if not os.path.exists((nwfolder)): os.makedirs(nwfolder) # TRIAL if t and not os.path.exists(fnamet): ibis_hit = [np.diff(D_trial[i][j]) for j in hits] ibis_hit_mat = np.full((len(hits), len(max(ibis_hit, key=len))), np.nan) for j in range(len(hits)): ibis_hit_mat[j, :len(ibis_hit[j])] = ibis_hit[j] ibis_miss = [np.diff(D_trial[i][j]) for j in misses] ibis_miss_mat = np.full( (len(misses), len(max(ibis_miss, key=len))), np.nan) for j in range(len(misses)): ibis_miss_mat[j, :len(ibis_miss[j])] = ibis_miss[j] fig = plt.figure(figsize=(20, 10)) hitsx = np.concatenate( [np.array(D_trial[i][j]) - array_end[j] for j in hits]) hitsy = np.concatenate( [np.full(len(D_trial[i][j]), j + 1) for j in hits]) missx = np.concatenate( [np.array(D_trial[i][j]) - array_end[j] for j in misses]) missy = np.concatenate( [np.full(len(D_trial[i][j]), j + 1) for j in misses]) gs = gridspec.GridSpec(3, 1, height_ratios=[5, 1, 1]) ax0 = plt.subplot(gs[0]) ax1 = plt.subplot(gs[1]) ax2 = plt.subplot(gs[2]) plt.subplots_adjust(hspace=0.5) ax0.plot(hitsx, hitsy, 'b.', markersize=3) ax0.plot(missx, missy, 'k.', markersize=3) ax0.plot(array_start - array_end, np.arange(1, len(array_start) + 1), 'r.', markersize=3) ax0.legend(['hits', 'miss', 'trial start']) ax0.axvline(0, color='r') ax0.set_title("PSTH trial") ax0.set_xlabel('Time(fr)') ax0.set_ylabel('Trial Number') blues, reds = sns.color_palette("Blues", 3), sns.color_palette('Reds', 3) for j, m in enumerate(metrics): t1 = IBI_cv_matrix(ibis_hit_mat, m) t2 = IBI_cv_matrix(ibis_miss_mat, m) ax1.plot(hits, (t1 - np.nanmin(t1)) / (np.nanmax(t1) - np.nanmin(t1)), c=blues[j], label='hit ' + m) ax1.plot(misses, (t2 - np.nanmin(t2)) / (np.nanmax(t2) - np.nanmin(t2)), c=reds[j], label='miss ' + m) ax1.set_title("HM cv evolution") ax1.set_ylabel('minmax scale') ax1.set_xlabel("Trial#") ax2.plot(hits, [np.nanmean(h) for h in ibis_hit]) ax2.plot(misses, [np.nanmean(mi) for mi in ibis_miss]) ax2.legend(['hit', 'miss']) ax2.set_xlabel("Trial#") ax2.set_ylabel("No. of Frames") #TODO: MAYBE ADD A PSTH here fig.savefig(fnamet + '.png') if eps: fig.savefig(fnamet + ".eps") plt.close('all') # WINDOW if w and not os.path.exists(fnamew): fig = plt.figure(figsize=(20, 10)) wlen = len(D_window[i]) slidex = np.concatenate( [np.array(D_window[i][j]) - window * j for j in range(wlen)]) slidey = np.concatenate( [np.full(len(D_window[i][j]), j + 1) for j in range(wlen)]) ibis_slide = [np.diff(D_window[i][j]) for j in range(wlen)] ibis_slide_mat = np.full((wlen, len(max(ibis_slide, key=len))), np.nan) for j in range(wlen): ibis_slide_mat[j, :len(ibis_slide[j])] = ibis_slide[j] gs = gridspec.GridSpec(3, 1, height_ratios=[5, 1, 1]) ax0 = plt.subplot(gs[0]) ax1 = plt.subplot(gs[1]) ax2 = plt.subplot(gs[2]) plt.subplots_adjust(hspace=0.5) ax0.plot(slidex, slidey, 'k.', markersize=3) ax0.axvline(0, color='r') ax0.set_title("PSTH window") ax0.set_xlabel('Time(fr)') ax0.set_ylabel('Slide') for m in metrics: temp = IBI_cv_matrix(ibis_slide_mat, m) ax1.plot(np.arange(wlen), (temp - np.nanmin(temp)) / (np.nanmax(temp) - np.nanmin(temp))) ax1.legend(metrics) ax1.set_title("CV evolution") ax1.set_ylabel('minmax scale') ax1.set_xlabel("Window") print(i, len(ibis_slide)) try: ax2.hist(ibis_slide, density=True, color=sns.color_palette("Blues", wlen)) except: print(ibis_slide) ax2.legend(np.arange(wlen)) ax2.set_title("IBI dist evolution") fig.savefig(fnamew + '.png') if eps: fig.savefig(fnamew + ".eps") plt.close("all")
def dff_sanity_check_single_session(rawbase, processed, animal, day, out=None, PROBELEN=1000, number_planes_total=6, mproc=False): rawpath = os.path.join(rawbase, animal, day) end = 10 onlinef = None for f in os.listdir(rawpath): if f.find('bmi_IntegrationRois') != -1: tend = int(f[-5]) if tend < end: end = tend onlinef = f if onlinef is None: raise FileNotFoundError('bmi_IntegrationRois') online_data = pd.read_csv(os.path.join(rawpath, onlinef)) f1 = os.path.join(processed, animal, "full_{}_{}__data.hdf5".format(animal, day)) f2 = encode_to_filename(processed, animal, day) if os.path.exists(f1): hfname = f1 elif os.path.exists(f2): hfname = f2 else: raise FileNotFoundError("File {} or {} not found".format(f1, f2)) with h5py.File(hfname, 'r') as hf: dff = np.array(hf['dff']) C = np.array(hf['C']) blen = hf.attrs['blen'] ens_neur = np.array(hf['ens_neur']) dff[np.isnan(dff)] = 0 dff_ens = dff[ens_neur] C_ens = C[ens_neur] units = len(ens_neur) N = 2 * units def helper(vars): R = np.corrcoef(vars) corrs_pair = np.diagonal(R, units) chance_corr = (np.nansum(R) / 2 - units - np.nansum(corrs_pair)) * 2 / (N ** 2 - 2 * N) return corrs_pair, chance_corr corrs_pair1, chance1 = helper(np.vstack([dff_ens, C_ens])) frames = online_data['frameNumber'].values // number_planes_total + blen online = online_data.iloc[:, 2:2 + units].values.T online[np.isnan(online)] = 0 slice_stack = np.vstack([dff_ens[:, frames], online]) corrs_pair2, chance2 = helper(slice_stack) b = [np.nan] * 4 corrs_pair3, chance3 = helper(np.vstack([C[:, frames], online])) if out is not None: CAIMANONLY = False OFFSET = 0 fig, axes = plt.subplots(2, 2, figsize=(20, 15)) axflat = axes.ravel() for i, ens in enumerate(ens_neur): if CAIMANONLY: axflat[i].plot(zscore(dff[ens])) axflat[i].plot(zscore(C[ens]) + OFFSET) axflat[i].legend(['CaImAnDFF', 'C']) axflat[i].set_title('Ens #{}'.format(ens)) else: # s = online_data.iloc[:, 2+i].values s = online[i] nmean = np.nanmean(s) auxonline = (s - nmean) / nmean onlinedff = auxonline onlineraw = s # TODO: get back to Nuria for the sample analysis plots in harddrive of the # ensemble vs C plots axflat[i].plot(zscore(dff[ens, frames[-PROBELEN:]])) axflat[i].plot(zscore(C[ens, frames[-PROBELEN:]]) + OFFSET * 1) # axflat[i].plot(zscore(onlinedff[-PROBELEN:]) + OFFSET * 2) axflat[i].plot(zscore(onlineraw[-PROBELEN:]) + OFFSET * 2) # axflat[i].legend(['CaImAnDFF', 'C', 'greedyDFF(f0=mean)', 'online raw']) axflat[i].legend(['CaImAnDFF', 'C', 'online raw']) axflat[i].set_title('Ens #{}'.format(ens)) fig.suptitle("CaImAn DFF Sanity Check {} {}{}".format(animal, day, " With Offset {}".format( OFFSET) if OFFSET else "")) basename = "dff_check_{}_{}{}{}".format(animal, day, "" if CAIMANONLY else "_with_raw_online", "_offset_{}".format(OFFSET) if OFFSET else "") tpath = os.path.join(out, "OFFSET{}".format(OFFSET)) if not os.path.exists(tpath): os.makedirs(tpath) outname = os.path.join(out, "OFFSET{}".format(OFFSET), basename) fig.savefig(outname + '.png') fig.savefig(outname + '.eps') if not mproc: plt.show() results = [animal, day, chance1] + b + [chance2] + b + [chance3] +b for i in range(units): results[i + 3] = corrs_pair1[i] results[i + 8] = corrs_pair2[i] results[i + 13] = corrs_pair3[i] return results
def ens_to_ind_GC_double_reconv_shuffle_test_single_session( folder, animal, day, test_reconv=True, thres=0.05, snr_thres=0, lagmode=2): """ Calculates granger causality from ensemble to indirect neurons :param ens_dff: E * T, E: number ensemble neurons :param ind_dff: I * T, I: number of indirect neurons :return: """ # get ens_dff, get_ind_dff utils = os.path.join(folder, 'utils') processed = os.path.join(folder, 'processed') with h5py.File(encode_to_filename(processed, animal, day), 'r') as hfile: roi_type = get_roi_type(hfile, animal, day) blen = hfile.attrs['blen'] e_sel = np.isin(roi_type, ['E1', 'E2', 'E']) inds = np.arange(len(roi_type)) ir_sel = roi_type == 'IR' ig_sel = roi_type == 'IG' dff = np.array(hfile['dff'])[:, :blen] dff_e = dff[e_sel, :] dff_ir = dff[ir_sel, :] dff_ig = dff[ig_sel, :] dff_inds = np.vstack([dff_ir, dff_ig]) GC_hf_inds = np.concatenate([inds[e_sel], inds[ir_sel], inds[ig_sel]]) if dff_e.shape[0] > 0: # get GC ens_dff to ind_dff pfname = encode_to_filename(utils, animal, day, hyperparams='granger') GC_inds, ens_to_I = get_ens_to_indirect_GC(pfname, roi_type, thres=0.05) shuf_file_found = False save_shufmat = {} try: shuf_file = encode_to_filename(utils, animal, day, hyperparams='reconv_shuffle') shuf_file_found = True shufmat = loadmat(shuf_file) dff_e_rshuffle = shufmat['dff_e_rshuffle'] dff_ind_rshuffle = shufmat['dff_ind_rshuffle'] except FileNotFoundError: # TODO: take care of rows of nans # reconvolve shuffle dff_e and dff_ind dff_e_rshuffle = np.vstack([ deconvolve_reconvolve(dff_e[i], shuffle=True) for i in range(dff_e.shape[0]) ]) dff_ind_rshuffle = np.vstack([ deconvolve_reconvolve(dff_inds[i], shuffle=True) for i in range(dff_inds.shape[0]) ]) save_shufmat['dff_e_rshuffle'] = dff_e_rshuffle save_shufmat['dff_ind_rshuffle'] = dff_ind_rshuffle save_shufmat['ens_to_I'] = ens_to_I save_shufmat['GC_inds'] = GC_inds dff_all_rshuffle = np.vstack([dff_e_rshuffle, dff_ind_rshuffle]) nansel_rshuffle = np.any(np.isnan(dff_all_rshuffle), axis=1) # calculate granger causality for the reconvolved data if isinstance(lagmode, str): try: lag = granger_select_order(dff_all_rshuffle[~nansel_rshuffle], maxlag=5, ic='bic') except: lag = 2 else: lag = lagmode gcs_val1, p_vals1 = statsmodel_granger_asymmetric( dff_e_rshuffle, dff_ind_rshuffle, lag, False) p_vals1 = p_vals1['ssr_chi2test'] gcs_val1[p_vals1 > thres] = 0 if not shuf_file_found: save_shufmat['gcs_rshuffle'] = gcs_val1 if lagmode == 'max': gcs_val1 = np.max(gcs_val1, axis=2) else: # auto or number gcs_val1 = gcs_val1[:, :, -1] assert gcs_val1.shape == ens_to_I.shape assert np.allclose(GC_hf_inds, GC_inds) # reconv comparison if test_reconv: # reconvolve dff_e and dff_ind if shuf_file_found: dff_e_reconv = shufmat['dff_e_reconv'] dff_ind_reconv = shufmat['dff_ind_reconv'] else: dff_e_reconv = np.vstack([ deconvolve_reconvolve(dff_e[i]) for i in range(dff_e.shape[0]) ]) dff_ind_reconv = np.vstack([ deconvolve_reconvolve(dff_inds[i]) for i in range(dff_inds.shape[0]) ]) save_shufmat['dff_e_reconv'] = dff_e_reconv save_shufmat['dff_ind_reconv'] = dff_ind_reconv dff_all_reconv = np.vstack([dff_e_reconv, dff_ind_reconv]) nansel_reconv = np.any(np.isnan(dff_all_reconv), axis=1) if isinstance(lagmode, str): try: lag = granger_select_order(dff_all_reconv[~nansel_reconv], maxlag=5, ic='bic') except: lag = 2 else: lag = lagmode gcs_val0, p_vals0 = statsmodel_granger_asymmetric( dff_e_reconv, dff_ind_reconv, lag, False) p_vals0 = p_vals0['ssr_chi2test'] gcs_val0[p_vals0 > thres] = 0 if not shuf_file_found: save_shufmat['gcs_reconv'] = gcs_val0 if lagmode == 'max': gcs_val0 = np.max(gcs_val0, axis=2) else: # auto or number gcs_val0 = gcs_val0[:, :, -1] assert gcs_val0.shape == ens_to_I.shape assert np.allclose(GC_hf_inds, GC_inds) snr_e = np.array([ caiman_SNR(None, dff_e[i], 'fast') for i in range(len(dff_e)) ]) snr_inds = np.array([ caiman_SNR(None, dff_inds[i], 'fast') for i in range(len(dff_inds)) ]) valid_sel_e = snr_e > snr_thres valid_sel_inds = snr_inds > snr_thres ens_to_I_valid = ens_to_I[valid_sel_e, :][:, valid_sel_inds] gcs_val0_valid = gcs_val0[valid_sel_e, :][:, valid_sel_inds] gcs_val1_valid = gcs_val1[valid_sel_e, :][:, valid_sel_inds] results = { 'dff_e_reconv': dff_e_reconv, 'dff_ind_reconv': dff_ind_reconv, 'dff_e_rshuffle': dff_e_rshuffle, 'dff_ind_rshuffle': dff_ind_reconv, 'ens_to_I': ens_to_I, 'GC_inds': GC_inds, 'gcs_rshuffle': gcs_val1, 'gcs_reconv': gcs_val0 } savemat( os.path.join(utils, 'FC/statsmodel/', animal, day, f'{animal}_{day}_reconv_shuffle_gc.mat'), results) fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(20, 10)) plt.subplots_adjust(hspace=0.4) visualize_gc_pairs(ens_to_I_valid, gcs_val0_valid, "GC", "reconvGC", axes[0], diff_label='d(raw,reconv)', verbose=True, firstbin=False, kde=False) visualize_gc_pairs(gcs_val0_valid, gcs_val1_valid, "reconvGC", "rshufGC", axes[1], diff_label='normalized', verbose=True, firstbin=False, kde=False) visualize_gc_pairs(ens_to_I_valid, gcs_val1_valid, "GC", "rshufGC", axes[2], diff_label='normalized', verbose=True, firstbin=False, kde=False) else: fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(20, 10)) visualize_gc_pairs(ens_to_I, gcs_val1, "GC", "rshufGC", axes, diff_label='normalized', verbose=True, firstbin=False, kde=False) if not shuf_file_found: savemat( encode_to_filename(utils, animal, day, hyperparams='reconv_shuffle', err=False), save_shufmat) else: raise RuntimeError(f"No ensemble neuron in {animal} {day}")