def pair_field_preprocess_exp(df, vthresh=5, sthresh=3, NShuffles=200, save_pth=None): pairdata_dict = dict(pair_id=[], ca=[], overlap=[], border1=[], border2=[], area1=[], area2=[], xyval1=[], xyval2=[], aver_rate1=[], aver_rate2=[], aver_rate_pair=[], peak_rate1=[], peak_rate2=[], minocc1=[], minocc2=[], fieldcoor1=[], fieldcoor2=[], com1=[], com2=[], rate_angle1=[], rate_angle2=[], rate_anglep=[], rate_R1=[], rate_R2=[], rate_Rp=[], num_spikes1=[], num_spikes2=[], num_spikes_pair=[], phaselag_AB=[], phaselag_BA=[], corr_info_AB=[], corr_info_BA=[], thetaT_AB=[], thetaT_BA=[], rate_AB=[], rate_BA=[], corate=[], pair_rate=[], kld=[], rate_R_pvalp=[], precess_df1=[], precess_angle1=[], precess_R1=[], precess_df2=[], precess_angle2=[], precess_R2=[], numpass_at_precess1=[], numpass_at_precess2=[], precess_dfp=[]) num_trials = df.shape[0] aedges = np.linspace(-np.pi, np.pi, 36) abind = aedges[1] - aedges[0] sp_binwidth = 5 aedges_precess = np.linspace(-np.pi, np.pi, 6) kappa_precess = 1 precess_filter = PrecessionFilter() pair_id = 0 for ntrial in range(num_trials): wave = df.loc[ntrial, 'wave'] precesser = PrecessionProcesser(wave=wave) for ca in ['CA%d' % (i + 1) for i in range(3)]: # Get data pair_df = df.loc[ntrial, ca + 'pairs'] field_df = df.loc[ntrial, ca + 'fields'] indata = df.loc[ntrial, ca + 'indata'] if (indata.shape[0] < 1) & (pair_df.shape[0] < 1) & (field_df.shape[0] < 1): continue tunner = IndataProcessor(indata, vthresh=vthresh, sthresh=sthresh, minpasstime=0.4) interpolater_angle = interp1d(tunner.t, tunner.angle) interpolater_x = interp1d(tunner.t, tunner.x) interpolater_y = interp1d(tunner.t, tunner.y) all_maxt, all_mint = tunner.t.max(), tunner.t.min() trange = (all_maxt, all_mint) dt = tunner.t[1] - tunner.t[0] precesser.set_trange(trange) ## Loop for pairs num_pairs = pair_df.shape[0] for npair in range(num_pairs): print('%d/%d trial, %s, %d/%d pair id=%d' % (ntrial, num_trials, ca, npair, num_pairs, pair_id)) # find within-mask indexes field_ids = pair_df.loc[npair, 'fi'] - 1 # minus 1 to convert to python index mask1, pf1, xyval1 = field_df.loc[field_ids[0], ['mask', 'pf', 'xyval']] mask2, pf2, xyval2 = field_df.loc[field_ids[1], ['mask', 'pf', 'xyval']] tsp1 = field_df.loc[field_ids[0], 'xytsp']['tsp'] tsp2 = field_df.loc[field_ids[1], 'xytsp']['tsp'] xaxis1, yaxis1 = pf1['X'][:, 0], pf1['Y'][0, :] xaxis2, yaxis2 = pf2['X'][:, 0], pf2['Y'][0, :] assert (np.all(xaxis1==xaxis2) and np.all(yaxis1==yaxis2)) area1 = mask1.sum() area2 = mask2.sum() field_d1 = np.sqrt(area1/np.pi)*2 field_d2 = np.sqrt(area2/np.pi)*2 # Find overlap _, ks_dist, _ = dist_overlap(pf1['map'], pf2['map'], mask1, mask2) # Field's center coordinates maskedmap1, maskedmap2 = pf1['map'] * mask1, pf2['map'] * mask2 cooridx1 = np.unravel_index(maskedmap1.argmax(), maskedmap1.shape) cooridx2 = np.unravel_index(maskedmap2.argmax(), maskedmap2.shape) fcoor1 = np.array([pf1['X'][cooridx1[0], cooridx1[1]], pf1['Y'][cooridx1[0], cooridx1[1]]]) fcoor2 = np.array([pf2['X'][cooridx2[0], cooridx2[1]], pf2['Y'][cooridx2[0], cooridx2[1]]]) XY1 = np.stack([pf1['X'].ravel(), pf1['Y'].ravel()]) XY2 = np.stack([pf2['X'].ravel(), pf2['Y'].ravel()]) com1 = np.sum(XY1 * (maskedmap1/np.sum(maskedmap1)).ravel().reshape(1, -1), axis=1) com2 = np.sum(XY2 * (maskedmap2/np.sum(maskedmap2)).ravel().reshape(1, -1), axis=1) # Border border1 = check_border(mask1, margin=2) border2 = check_border(mask2, margin=2) # Construct passes (segment & chunk) for field 1 and 2 tok1, idin1 = tunner.get_idin(mask1, xaxis1, yaxis1) tok2, idin2 = tunner.get_idin(mask2, xaxis2, yaxis2) passdf1 = tunner.construct_singlefield_passdf(tok1, tsp1, interpolater_x, interpolater_y, interpolater_angle) passdf2 = tunner.construct_singlefield_passdf(tok2, tsp2, interpolater_x, interpolater_y, interpolater_angle) allchunk_df1 = passdf1[(~passdf1['rejected']) & (passdf1['chunked']<2)].reset_index(drop=True) allchunk_df2 = passdf2[(~passdf2['rejected']) & (passdf2['chunked']<2)].reset_index(drop=True) # allchunk_df1 = passdf1[(~passdf1['rejected']) ].reset_index(drop=True) # allchunk_df2 = passdf2[(~passdf2['rejected']) ].reset_index(drop=True) if (allchunk_df1.shape[0] < 1) or (allchunk_df2.shape[0] < 1): continue x1list, y1list, angle1list = allchunk_df1['x'].to_list(), allchunk_df1['y'].to_list(), allchunk_df1['angle'].to_list() t1list, tsp1list = allchunk_df1['t'].to_list(), allchunk_df1['tsp'].to_list() x2list, y2list, angle2list = allchunk_df2['x'].to_list(), allchunk_df2['y'].to_list(), allchunk_df2['angle'].to_list() t2list, tsp2list = allchunk_df2['t'].to_list(), allchunk_df2['tsp'].to_list() if (len(t1list) < 1) or (len(t2list) < 1) or (len(tsp1list) < 1) or (len(tsp2list) < 1): continue x1, x2 = np.concatenate(x1list), np.concatenate(x2list) y1, y2 = np.concatenate(y1list), np.concatenate(y2list) hd1, hd2 = np.concatenate(angle1list), np.concatenate(angle2list) pos1, pos2 = np.stack([x1, y1]).T, np.stack([x2, y2]).T tsp1, tsp2 = np.concatenate(tsp1list), np.concatenate(tsp2list) xsp1, xsp2 = np.concatenate(allchunk_df1['spikex'].to_list()), np.concatenate(allchunk_df2['spikex'].to_list()) ysp1, ysp2 = np.concatenate(allchunk_df1['spikey'].to_list()), np.concatenate(allchunk_df2['spikey'].to_list()) possp1, possp2 = np.stack([xsp1, ysp1]).T, np.stack([xsp2, ysp2]).T hdsp1 = np.concatenate(allchunk_df1['spikeangle'].to_list()) hdsp2 = np.concatenate(allchunk_df2['spikeangle'].to_list()) nspks1, nspks2 = tsp1.shape[0], tsp2.shape[0] # Rates aver_rate1 = nspks1 / (x1.shape[0]*dt) peak_rate1 = np.max(pf1['map'] * mask1) aver_rate2 = nspks2 / (x2.shape[0] * dt) peak_rate2 = np.max(pf2['map'] * mask2) # Directionality occbins1, _ = np.histogram(hd1, bins=aedges) spbins1, _ = np.histogram(hdsp1, bins=aedges) mlmer1 = DirectionerMLM(pos1, hd1, dt=dt, sp_binwidth=sp_binwidth, a_binwidth=abind) rate_angle1, rate_R1, normprob1_mlm = mlmer1.get_directionality(possp1, hdsp1) normprob1_mlm[np.isnan(normprob1_mlm)] = 0 occbins2, _ = np.histogram(hd2, bins=aedges) spbins2, _ = np.histogram(hdsp2, bins=aedges) mlmer2 = DirectionerMLM(pos2, hd2, dt=dt, sp_binwidth=sp_binwidth, a_binwidth=abind) rate_angle2, rate_R2, normprob2_mlm = mlmer2.get_directionality(possp2, hdsp2) normprob2_mlm[np.isnan(normprob2_mlm)] = 0 minocc1, minocc2 = occbins1.min(), occbins2.min() neuro_keys_dict = dict(tsp='tsp', spikev='spikev', spikex='spikex', spikey='spikey', spikeangle='spikeangle') # Precession1 & Post-hoc exclusion for 1st field accept_mask1 = (~passdf1['rejected']) & (passdf1['chunked'] < 2) passdf1['excluded_for_precess'] = ~accept_mask1 precessdf1, precessangle1, precessR1, _ = get_single_precessdf(passdf1, precesser, precess_filter, neuro_keys_dict, field_d=field_d1, kappa=kappa_precess, bins=None) fitted_precessdf1 = precessdf1[precessdf1['fitted']].reset_index(drop=True) if (precessangle1 is not None) and (fitted_precessdf1.shape[0] > 0): # Post-hoc precession exclusion _, _, postdoc_dens1 = compute_precessangle(pass_angles=fitted_precessdf1['mean_anglesp'].to_numpy(), pass_nspikes=fitted_precessdf1['pass_nspikes'].to_numpy(), precess_mask=fitted_precessdf1['precess_exist'].to_numpy(), kappa=None, bins=aedges_precess) (_, passbins_p1, passbins_np1, _) = postdoc_dens1 all_passbins1 = passbins_p1 + passbins_np1 numpass_at_precess1 = get_numpass_at_angle(target_angle=precessangle1, aedge=aedges_precess, all_passbins=all_passbins1) else: numpass_at_precess1 = None # Precession2 & Post-hoc exclusion for 2nd field accept_mask2 = (~passdf2['rejected']) & (passdf2['chunked'] < 2) passdf2['excluded_for_precess'] = ~accept_mask2 precessdf2, precessangle2, precessR2, _ = get_single_precessdf(passdf2, precesser, precess_filter, neuro_keys_dict, field_d=field_d2, kappa=kappa_precess, bins=None) fitted_precessdf2 = precessdf2[precessdf2['fitted']].reset_index(drop=True) if (precessangle2 is not None) and (fitted_precessdf2.shape[0] > 0): # Post-hoc precession exclusion _, _, postdoc_dens2 = compute_precessangle(pass_angles=fitted_precessdf2['mean_anglesp'].to_numpy(), pass_nspikes=fitted_precessdf2['pass_nspikes'].to_numpy(), precess_mask=fitted_precessdf2['precess_exist'].to_numpy(), kappa=None, bins=aedges_precess) (_, passbins_p2, passbins_np2, _) = postdoc_dens2 all_passbins2 = passbins_p2 + passbins_np2 numpass_at_precess2 = get_numpass_at_angle(target_angle=precessangle2, aedge=aedges_precess, all_passbins=all_passbins2) else: numpass_at_precess2 = None # # Paired field processing # Construct pairedpasses mask_union = mask1 | mask2 field_d_union = np.sqrt(mask_union.sum()/np.pi)*2 tok_pair, _ = tunner.get_idin(mask_union, xaxis1, yaxis1) pairedpasses = tunner.construct_pairfield_passdf(tok_pair, tok1, tok2, tsp1, tsp2, interpolater_x, interpolater_y, interpolater_angle) # Phase lags phase_finder = ThetaEstimator(0.005, 0.3, [5, 12]) AB_tsp1_list, BA_tsp1_list = [], [] AB_tsp2_list, BA_tsp2_list = [], [] nspikes_AB_list, nspikes_BA_list = [], [] duration_AB_list, duration_BA_list = [], [] t_all = [] passangles_all, x_all, y_all = [], [], [] paired_tsp_list = [] accepted_df = pairedpasses[(~pairedpasses['rejected']) & (pairedpasses['chunked']<2)].reset_index(drop=True) # accepted_df = pairedpasses[(~pairedpasses['rejected'])].reset_index(drop=True) for npass in range(accepted_df.shape[0]): # Get data t, tsp1, tsp2 = accepted_df.loc[npass, ['t', 'tsp1', 'tsp2']] x, y, pass_angles, v, direction = accepted_df.loc[npass, ['x', 'y', 'angle', 'v', 'direction']] duration = t.max() - t.min() # Find paired spikes pairidx1, pairidx2 = find_pair_times(tsp1, tsp2) paired_tsp1, paired_tsp2 = tsp1[pairidx1], tsp2[pairidx2] if (paired_tsp1.shape[0] < 1) and (paired_tsp2.shape[0] < 1): continue paired_tsp_eachpass = np.concatenate([paired_tsp1, paired_tsp2]) paired_tsp_list.append(paired_tsp_eachpass) passangles_all.append(pass_angles) x_all.append(x) y_all.append(y) t_all.append(t) if direction == 'A->B': AB_tsp1_list.append(tsp1) AB_tsp2_list.append(tsp2) nspikes_AB_list.append(tsp1.shape[0] + tsp2.shape[0]) duration_AB_list.append(duration) elif direction == 'B->A': BA_tsp1_list.append(tsp1) BA_tsp2_list.append(tsp2) nspikes_BA_list.append(tsp1.shape[0] + tsp2.shape[0]) duration_BA_list.append(duration) # Phase lags thetaT_AB, phaselag_AB, corr_info_AB = phase_finder.find_theta_isi_hilbert(AB_tsp1_list, AB_tsp2_list) thetaT_BA, phaselag_BA, corr_info_BA = phase_finder.find_theta_isi_hilbert(BA_tsp1_list, BA_tsp2_list) # Pair precession neuro_keys_dict1 = dict(tsp='tsp1', spikev='spike1v', spikex='spike1x', spikey='spike1y', spikeangle='spike1angle') neuro_keys_dict2 = dict(tsp='tsp2', spikev='spike2v', spikex='spike2x', spikey='spike2y', spikeangle='spike2angle') accept_mask = (~pairedpasses['rejected']) & (pairedpasses['chunked']<2) & ((pairedpasses['direction']=='A->B')| (pairedpasses['direction']=='B->A')) pairedpasses['excluded_for_precess'] = ~accept_mask precess_dfp = precesser.get_single_precession(pairedpasses, neuro_keys_dict1, field_d_union, tag='1') precess_dfp = precesser.get_single_precession(precess_dfp, neuro_keys_dict2, field_d_union, tag='2') precess_dfp = precess_filter.filter_pair(precess_dfp) fitted_precess_dfp = precess_dfp[precess_dfp['fitted1'] & precess_dfp['fitted2']].reset_index(drop=True) # Paired spikes if (len(paired_tsp_list) == 0) or (len(passangles_all) == 0): continue hd_pair = np.concatenate(passangles_all) x_pair, y_pair = np.concatenate(x_all), np.concatenate(y_all) pos_pair = np.stack([x_pair, y_pair]).T paired_tsp = np.concatenate(paired_tsp_list) paired_tsp = paired_tsp[(paired_tsp <= all_maxt) & (paired_tsp >= all_mint)] if paired_tsp.shape[0] < 1: continue num_spikes_pair = paired_tsp.shape[0] hdsp_pair = interpolater_angle(paired_tsp) xsp_pair = interpolater_x(paired_tsp) ysp_pair = interpolater_y(paired_tsp) possp_pair = np.stack([xsp_pair, ysp_pair]).T aver_rate_pair = num_spikes_pair / (x_pair.shape[0] * dt) # Pair Directionality occbinsp, _ = np.histogram(hd_pair, bins=aedges) spbinsp, _ = np.histogram(hdsp_pair, bins=aedges) mlmer_pair = DirectionerMLM(pos_pair, hd_pair, dt, sp_binwidth, abind) rate_anglep, rate_Rp, normprobp_mlm = mlmer_pair.get_directionality(possp_pair, hdsp_pair) normprobp_mlm[np.isnan(normprobp_mlm)] = 0 # Time shift shuffling if np.isnan(rate_Rp): rate_R_pvalp = np.nan else: rate_R_pvalp = timeshift_shuffle_exp_wrapper(paired_tsp_list, t_all, rate_Rp, NShuffles, mlmer_pair, interpolater_x, interpolater_y, interpolater_angle, trange) # Rates with np.errstate(divide='ignore', invalid='ignore'): # None means no sample rate_AB = np.sum(nspikes_AB_list) / np.sum(duration_AB_list) rate_BA = np.sum(nspikes_BA_list) / np.sum(duration_BA_list) corate = np.sum(nspikes_AB_list + nspikes_BA_list) / np.sum(duration_AB_list + duration_BA_list) pair_rate = num_spikes_pair / np.sum(duration_AB_list + duration_BA_list) # KLD kld = calc_kld(normprob1_mlm, normprob2_mlm, normprobp_mlm) pairdata_dict['pair_id'].append(pair_id) pairdata_dict['ca'].append(ca) pairdata_dict['overlap'].append(ks_dist) pairdata_dict['border1'].append(border1) pairdata_dict['border2'].append(border2) pairdata_dict['area1'].append(area1) pairdata_dict['area2'].append(area2) pairdata_dict['xyval1'].append(xyval1) pairdata_dict['xyval2'].append(xyval2) pairdata_dict['aver_rate1'].append(aver_rate1) pairdata_dict['aver_rate2'].append(aver_rate2) pairdata_dict['aver_rate_pair'].append(aver_rate_pair) pairdata_dict['peak_rate1'].append(peak_rate1) pairdata_dict['peak_rate2'].append(peak_rate2) pairdata_dict['minocc1'].append(minocc1) pairdata_dict['minocc2'].append(minocc2) pairdata_dict['fieldcoor1'].append(fcoor1) pairdata_dict['fieldcoor2'].append(fcoor2) pairdata_dict['com1'].append(com1) pairdata_dict['com2'].append(com2) pairdata_dict['rate_angle1'].append(rate_angle1) pairdata_dict['rate_angle2'].append(rate_angle2) pairdata_dict['rate_anglep'].append(rate_anglep) pairdata_dict['rate_R1'].append(rate_R1) pairdata_dict['rate_R2'].append(rate_R2) pairdata_dict['rate_Rp'].append(rate_Rp) pairdata_dict['num_spikes1'].append(nspks1) pairdata_dict['num_spikes2'].append(nspks2) pairdata_dict['num_spikes_pair'].append(num_spikes_pair) pairdata_dict['phaselag_AB'].append(phaselag_AB) pairdata_dict['phaselag_BA'].append(phaselag_BA) pairdata_dict['corr_info_AB'].append(corr_info_AB) pairdata_dict['corr_info_BA'].append(corr_info_BA) pairdata_dict['thetaT_AB'].append(thetaT_AB) pairdata_dict['thetaT_BA'].append(thetaT_BA) pairdata_dict['rate_AB'].append(rate_AB) pairdata_dict['rate_BA'].append(rate_BA) pairdata_dict['corate'].append(corate) pairdata_dict['pair_rate'].append(pair_rate) pairdata_dict['kld'].append(kld) pairdata_dict['rate_R_pvalp'].append(rate_R_pvalp) pairdata_dict['precess_df1'].append(fitted_precessdf1) pairdata_dict['precess_angle1'].append(precessangle1) pairdata_dict['precess_R1'].append(precessR1) pairdata_dict['numpass_at_precess1'].append(numpass_at_precess1) pairdata_dict['precess_df2'].append(fitted_precessdf2) pairdata_dict['precess_angle2'].append(precessangle2) pairdata_dict['precess_R2'].append(precessR2) pairdata_dict['numpass_at_precess2'].append(numpass_at_precess2) pairdata_dict['precess_dfp'].append(fitted_precess_dfp) pair_id += 1 pairdata = pd.DataFrame(pairdata_dict) pairdata = append_extrinsicity(pairdata) pairdata.to_pickle(save_pth) return pairdata
def single_field_preprocess_networks(simdata, radius=10, vthresh=2, sthresh=80, NShuffles=200, subsample_fraction=1, save_pth=None): """ Parameters ---------- Indata SpikeData NeuronPos radius vthresh : float Default = 5. Same as Emily's data. sthresh : float Default = 80. Determined by the same percentile (10%) of passes excluded in Emily's data (sthresh = 3 there). subsample_fraction : float The fraction that the spikes would be subsampled. =1 if no subsampling is needed Returns ------- """ datadict = dict(num_spikes=[], border=[], aver_rate=[], peak_rate=[], fieldangle_mlm=[], fieldR_mlm=[], spike_bins=[], occ_bins=[], shift_pval_mlm=[], precess_df=[], precess_angle=[], precess_angle_low=[], precess_R=[]) Indata, SpikeData, NeuronPos = simdata['Indata'], simdata[ 'SpikeData'], simdata['NeuronPos'] aedges = np.linspace(-np.pi, np.pi, 36) abind = aedges[1] - aedges[0] sp_binwidth = 5 tunner = IndataProcessor(Indata, vthresh, True) wave = dict(tax=Indata['t'].to_numpy(), phase=Indata['phase'].to_numpy(), theta=np.ones(Indata.shape[0])) interpolater_angle = interp1d(tunner.t, tunner.angle) interpolater_x = interp1d(tunner.t, tunner.x) interpolater_y = interp1d(tunner.t, tunner.y) trange = (tunner.t.max(), tunner.t.min()) dt = tunner.t[1] - tunner.t[0] precesser = PrecessionProcesser(sthresh=sthresh, vthresh=vthresh, wave=wave) precesser.set_trange(trange) precess_filter = PrecessionFilter() nspikes_stats = (13, 40) pass_nspikes = [] num_neurons = NeuronPos.shape[0] for nidx in range(num_neurons): print('%d/%d Neuron' % (nidx, num_neurons)) # Get spike indexes + subsample spdf = SpikeData[SpikeData['neuronidx'] == nidx].reset_index(drop=True) tidxsp = spdf['tidxsp'].to_numpy().astype(int) if subsample_fraction < 1: np.random.seed(nidx) sampled_tidxsp = np.random.choice(tidxsp.shape[0] - 1, int(tidxsp.shape[0] * subsample_fraction), replace=False) sampled_tidxsp.sort() tidxsp = tidxsp[sampled_tidxsp] # Get tok neuronx, neurony = NeuronPos.loc[nidx, ['neuronx', 'neurony']] dist = np.sqrt((neuronx - tunner.x)**2 + (neurony - tunner.y)**2) tok = dist < radius # Check border border = check_border_sim(neuronx, neurony, radius, (-40, 40)) # Get info from passes all_passidx = segment_passes(tok) passdf = construct_passdf_sim(tunner, all_passidx, tidxsp) beh_info, all_tsp_list = append_info_from_passes( passdf, vthresh, sthresh, trange) (all_x_list, all_y_list, all_t_list, all_passangles_list) = beh_info if (len(all_tsp_list) < 1) or (len(all_x_list) < 1): continue all_x = np.concatenate(all_x_list) all_y = np.concatenate(all_y_list) all_passangles = np.concatenate(all_passangles_list) all_tsp = np.concatenate(all_tsp_list) all_anglesp = interpolater_angle(all_tsp) xsp, ysp = interpolater_x(all_tsp), interpolater_y(all_tsp) pos = np.stack([all_x, all_y]).T possp = np.stack([xsp, ysp]).T # Average firing rate aver_rate = all_tsp.shape[0] / (all_x.shape[0] * dt) # Field's directionality num_spikes = all_tsp.shape[0] occ_bins, _ = np.histogram(all_passangles, bins=aedges) spike_bins, _ = np.histogram(all_anglesp, bins=aedges) mlmer = DirectionerMLM(pos, all_passangles, dt, sp_binwidth=sp_binwidth, a_binwidth=abind) fieldangle_mlm, fieldR_mlm, norm_prob_mlm = mlmer.get_directionality( possp, all_anglesp) # Precession per pass neuro_keys_dict = dict(tsp='tsp', spikev='spikev', spikex='spikex', spikey='spikey', spikeangle='spikeangle') precessdf, precess_angle, precess_R, _ = get_single_precessdf( passdf, precesser, precess_filter, neuro_keys_dict) filtered_precessdf = precessdf[precessdf['precess_exist']].reset_index( drop=True) pass_nspikes = pass_nspikes + list(filtered_precessdf['pass_nspikes']) # Precession - pass with low/high spike number if precessdf.shape[0] > 0: ldf = precessdf[precessdf['pass_nspikes'] < nspikes_stats[0]] # 25% quantile if (ldf.shape[0] > 0) and (ldf['precess_exist'].sum() > 0): passangle_l, pass_nspikes_l = ldf['spike_angle'].to_numpy( ), ldf['pass_nspikes'].to_numpy() precess_mask_l = ldf['precess_exist'].to_numpy() precess_angle_low, _, _ = compute_precessangle( passangle_l, pass_nspikes_l, precess_mask_l) else: precess_angle_low = None else: precess_angle_low = None # Time shift shuffling shi_pval_mlm = timeshift_shuffle_exp_wrapper( all_tsp_list, all_t_list, fieldR_mlm, NShuffles, mlmer, interpolater_x, interpolater_y, interpolater_angle, trange) datadict['border'].append(border) datadict['num_spikes'].append(num_spikes) datadict['aver_rate'].append(aver_rate) datadict['peak_rate'].append(None) datadict['fieldangle_mlm'].append(fieldangle_mlm) datadict['fieldR_mlm'].append(fieldR_mlm) datadict['spike_bins'].append(spike_bins) datadict['occ_bins'].append(occ_bins) datadict['shift_pval_mlm'].append(shi_pval_mlm) datadict['precess_df'].append(precessdf) datadict['precess_angle'].append(precess_angle) datadict['precess_angle_low'].append(precess_angle_low) datadict['precess_R'].append(precess_R) print('Num spikes:\n') print( pd.DataFrame({'pass_nspikes': pass_nspikes})['pass_nspikes'].describe()) datadf = pd.DataFrame(datadict) datadf.to_pickle(save_pth)
def pair_field_preprocess_Romani(simdata, save_pth, radius=2, vthresh=2, sthresh=80, NShuffles=200): pairdata_dict = dict( neuron1id=[], neuron2id=[], neuron1pos=[], neuron2pos=[], neurondist=[], # overlap is calculated afterward border1=[], border2=[], aver_rate1=[], aver_rate2=[], aver_rate_pair=[], com1=[], com2=[], rate_angle1=[], rate_angle2=[], rate_anglep=[], rate_R1=[], rate_R2=[], rate_Rp=[], num_spikes1=[], num_spikes2=[], num_spikes_pair=[], phaselag_AB=[], phaselag_BA=[], corr_info_AB=[], corr_info_BA=[], rate_AB=[], rate_BA=[], corate=[], pair_rate=[], kld=[], rate_R_pvalp=[], precess_df1=[], precess_angle1=[], precess_R1=[], precess_df2=[], precess_angle2=[], precess_R2=[], numpass_at_precess1=[], numpass_at_precess2=[], precess_dfp=[]) Indata, SpikeData, NeuronPos = simdata['Indata'], simdata[ 'SpikeData'], simdata['NeuronPos'] wave = dict(tax=Indata['t'].to_numpy(), phase=Indata['phase'].to_numpy(), theta=np.ones(Indata.shape[0])) subsample_fraction = 0.25 # setting minpasstime = 0.4 minspiketresh = 14 default_T = 1 / 10 aedges = np.linspace(-np.pi, np.pi, 36) abind = aedges[1] - aedges[0] sp_binwidth = 2 * np.pi / 16 precesser = PrecessionProcesser(wave=wave) precess_filter = PrecessionFilter() kappa_precess = 1 aedges_precess = np.linspace(-np.pi, np.pi, 6) field_d = radius * 2 # Precomputation tunner = IndataProcessor(Indata, vthresh=vthresh, sthresh=sthresh, minpasstime=minpasstime) interpolater_angle = interp1d(tunner.t, tunner.angle) interpolater_x = interp1d(tunner.t, tunner.x) interpolater_y = interp1d(tunner.t, tunner.y) all_maxt, all_mint = tunner.t.max(), tunner.t.min() trange = (all_maxt, all_mint) dt = tunner.t[1] - tunner.t[0] precesser.set_trange(trange) # Combination of pairs np.random.seed(0) sampled_idx_all = np.random.choice(NeuronPos.shape[0], 200, replace=False).astype(int) samples1 = sampled_idx_all[0:80] samples2 = sampled_idx_all[80:160] total_num = samples1.shape[0] * samples2.shape[0] progress = 0 for i in samples1: for j in samples2: print( '%d/%d (%d,%d): sample %d' % (progress, total_num, i, j, len(pairdata_dict['neuron1pos']))) # Distance neu1x, neu1y = NeuronPos.iloc[i] neu2x, neu2y = NeuronPos.iloc[j] neurondist = np.sqrt((neu1x - neu2x)**2 + (neu1y - neu2y)**2) # Get tok tok1 = np.sqrt((tunner.x - neu1x)**2 + (tunner.y - neu1y)**2) < radius tok2 = np.sqrt((tunner.x - neu2x)**2 + (tunner.y - neu2y)**2) < radius tok_union = tok1 | tok2 tok_intersect = tok1 & tok2 if np.sum(tok_intersect) < 10: # anything larger than 0 will do progress += 1 continue # Get spike indexes + subsample spdf1 = SpikeData[SpikeData['neuronidx'] == i].reset_index( drop=True) spdf2 = SpikeData[SpikeData['neuronidx'] == j].reset_index( drop=True) tidxsp1, tidxsp2 = spdf1['tidxsp'].to_numpy().astype( int), spdf2['tidxsp'].to_numpy().astype(int) np.random.seed(i * j) sampled_tidxsp1 = np.random.choice(tidxsp1.shape[0] - 1, int(tidxsp1.shape[0] * subsample_fraction), replace=False) sampled_tidxsp1.sort() tidxsp1 = tidxsp1[sampled_tidxsp1] np.random.seed(i * j + 1) sampled_tidxsp2 = np.random.choice(tidxsp2.shape[0] - 1, int(tidxsp2.shape[0] * subsample_fraction), replace=False) sampled_tidxsp2.sort() tidxsp2 = tidxsp2[sampled_tidxsp2] tsp1 = tunner.t[tidxsp1] tsp2 = tunner.t[tidxsp2] # Condition: Paired spikes exist tsp_diff = pair_diff(tsp1, tsp2) spok = np.sum(np.abs(tsp_diff) < default_T) if spok < minspiketresh: progress = progress + 1 continue # Check border border1 = check_border_sim(neu1x, neu1y, radius, (0, 2 * np.pi)) border2 = check_border_sim(neu2x, neu2y, radius, (0, 2 * np.pi)) # Construct passes (segment & chunk) for field 1 and 2 passdf1 = tunner.construct_singlefield_passdf( tok1, tsp1, interpolater_x, interpolater_y, interpolater_angle) passdf2 = tunner.construct_singlefield_passdf( tok2, tsp2, interpolater_x, interpolater_y, interpolater_angle) # allchunk_df1 = passdf1[(~passdf1['rejected']) & (passdf1['chunked']<2)].reset_index(drop=True) # allchunk_df2 = passdf2[(~passdf2['rejected']) & (passdf2['chunked']<2)].reset_index(drop=True) allchunk_df1 = passdf1[(~passdf1['rejected'])].reset_index( drop=True) allchunk_df2 = passdf2[(~passdf2['rejected'])].reset_index( drop=True) if (allchunk_df1.shape[0] < 1) or (allchunk_df2.shape[0] < 1): continue x1list, y1list, angle1list = allchunk_df1['x'].to_list( ), allchunk_df1['y'].to_list(), allchunk_df1['angle'].to_list() t1list, tsp1list = allchunk_df1['t'].to_list( ), allchunk_df1['tsp'].to_list() x2list, y2list, angle2list = allchunk_df2['x'].to_list( ), allchunk_df2['y'].to_list(), allchunk_df2['angle'].to_list() t2list, tsp2list = allchunk_df2['t'].to_list( ), allchunk_df2['tsp'].to_list() if (len(t1list) < 1) or (len(t2list) < 1) or ( len(tsp1list) < 1) or (len(tsp2list) < 1): continue x1, x2 = np.concatenate(x1list), np.concatenate(x2list) y1, y2 = np.concatenate(y1list), np.concatenate(y2list) hd1, hd2 = np.concatenate(angle1list), np.concatenate(angle2list) pos1, pos2 = np.stack([x1, y1]).T, np.stack([x2, y2]).T tsp1, tsp2 = np.concatenate(tsp1list), np.concatenate(tsp2list) xsp1, xsp2 = np.concatenate( allchunk_df1['spikex'].to_list()), np.concatenate( allchunk_df2['spikex'].to_list()) ysp1, ysp2 = np.concatenate( allchunk_df1['spikey'].to_list()), np.concatenate( allchunk_df2['spikey'].to_list()) possp1, possp2 = np.stack([xsp1, ysp1]).T, np.stack([xsp2, ysp2]).T hdsp1 = np.concatenate(allchunk_df1['spikeangle'].to_list()) hdsp2 = np.concatenate(allchunk_df2['spikeangle'].to_list()) nspks1, nspks2 = tsp1.shape[0], tsp2.shape[0] # Rates aver_rate1 = nspks1 / (x1.shape[0] * dt) peak_rate1 = None aver_rate2 = nspks2 / (x2.shape[0] * dt) peak_rate2 = None # Directionality occbins1, _ = np.histogram(hd1, bins=aedges) spbins1, _ = np.histogram(hdsp1, bins=aedges) mlmer1 = DirectionerMLM(pos1, hd1, dt=dt, sp_binwidth=sp_binwidth, a_binwidth=abind) rate_angle1, rate_R1, normprob1_mlm = mlmer1.get_directionality( possp1, hdsp1) normprob1_mlm[np.isnan(normprob1_mlm)] = 0 occbins2, _ = np.histogram(hd2, bins=aedges) spbins2, _ = np.histogram(hdsp2, bins=aedges) mlmer2 = DirectionerMLM(pos2, hd2, dt=dt, sp_binwidth=sp_binwidth, a_binwidth=abind) rate_angle2, rate_R2, normprob2_mlm = mlmer2.get_directionality( possp2, hdsp2) normprob2_mlm[np.isnan(normprob2_mlm)] = 0 neuro_keys_dict = dict(tsp='tsp', spikev='spikev', spikex='spikex', spikey='spikey', spikeangle='spikeangle') # Precession1 & Post-hoc exclusion for 1st field accept_mask1 = (~passdf1['rejected']) & (passdf1['chunked'] < 2) passdf1['excluded_for_precess'] = ~accept_mask1 precessdf1, precessangle1, precessR1, _ = get_single_precessdf( passdf1, precesser, precess_filter, neuro_keys_dict, field_d=field_d, kappa=kappa_precess, bins=None) fitted_precessdf1 = precessdf1[precessdf1['fitted']].reset_index( drop=True) if (precessangle1 is not None) and (fitted_precessdf1.shape[0] > 0): # Post-hoc precession exclusion _, _, postdoc_dens1 = compute_precessangle( pass_angles=fitted_precessdf1['mean_anglesp'].to_numpy(), pass_nspikes=fitted_precessdf1['pass_nspikes'].to_numpy(), precess_mask=fitted_precessdf1['precess_exist'].to_numpy(), kappa=None, bins=aedges_precess) (_, passbins_p1, passbins_np1, _) = postdoc_dens1 all_passbins1 = passbins_p1 + passbins_np1 numpass_at_precess1 = get_numpass_at_angle( target_angle=precessangle1, aedge=aedges_precess, all_passbins=all_passbins1) else: numpass_at_precess1 = None # Precession2 & Post-hoc exclusion for 2nd field accept_mask2 = (~passdf2['rejected']) & (passdf2['chunked'] < 2) passdf2['excluded_for_precess'] = ~accept_mask2 precessdf2, precessangle2, precessR2, _ = get_single_precessdf( passdf2, precesser, precess_filter, neuro_keys_dict, field_d=field_d, kappa=kappa_precess, bins=None) fitted_precessdf2 = precessdf2[precessdf2['fitted']].reset_index( drop=True) if (precessangle2 is not None) and (fitted_precessdf2.shape[0] > 0): # Post-hoc precession exclusion _, _, postdoc_dens2 = compute_precessangle( pass_angles=fitted_precessdf2['mean_anglesp'].to_numpy(), pass_nspikes=fitted_precessdf2['pass_nspikes'].to_numpy(), precess_mask=fitted_precessdf2['precess_exist'].to_numpy(), kappa=None, bins=aedges_precess) (_, passbins_p2, passbins_np2, _) = postdoc_dens2 all_passbins2 = passbins_p2 + passbins_np2 numpass_at_precess2 = get_numpass_at_angle( target_angle=precessangle2, aedge=aedges_precess, all_passbins=all_passbins2) else: numpass_at_precess2 = None # # Paired field processing field_d_union = radius * 2 pairedpasses = tunner.construct_pairfield_passdf( tok_union, tok1, tok2, tsp1, tsp2, interpolater_x, interpolater_y, interpolater_angle) phase_finder = ThetaEstimator(0.005, 0.3, [5, 12]) AB_tsp1_list, BA_tsp1_list = [], [] AB_tsp2_list, BA_tsp2_list = [], [] nspikes_AB_list, nspikes_BA_list = [], [] duration_AB_list, duration_BA_list = [], [] t_all = [] passangles_all, x_all, y_all = [], [], [] paired_tsp_list = [] # accepted_df = pairedpasses[(~pairedpasses['rejected']) & (pairedpasses['chunked']<2)].reset_index(drop=True) accepted_df = pairedpasses[( ~pairedpasses['rejected'])].reset_index(drop=True) for npass in range(accepted_df.shape[0]): # Get data t, tsp1, tsp2 = accepted_df.loc[npass, ['t', 'tsp1', 'tsp2']] x, y, pass_angles, v, direction = accepted_df.loc[ npass, ['x', 'y', 'angle', 'v', 'direction']] duration = t.max() - t.min() # Find paired spikes pairidx1, pairidx2 = find_pair_times(tsp1, tsp2) paired_tsp1, paired_tsp2 = tsp1[pairidx1], tsp2[pairidx2] if (paired_tsp1.shape[0] < 1) and (paired_tsp2.shape[0] < 1): continue paired_tsp_eachpass = np.concatenate( [paired_tsp1, paired_tsp2]) paired_tsp_list.append(paired_tsp_eachpass) passangles_all.append(pass_angles) x_all.append(x) y_all.append(y) t_all.append(t) if direction == 'A->B': AB_tsp1_list.append(tsp1) AB_tsp2_list.append(tsp2) nspikes_AB_list.append(tsp1.shape[0] + tsp2.shape[0]) duration_AB_list.append(duration) elif direction == 'B->A': BA_tsp1_list.append(tsp1) BA_tsp2_list.append(tsp2) nspikes_BA_list.append(tsp1.shape[0] + tsp2.shape[0]) duration_BA_list.append(duration) # Phase lags thetaT_AB, phaselag_AB, corr_info_AB = phase_finder.find_theta_isi_hilbert( AB_tsp1_list, AB_tsp2_list) thetaT_BA, phaselag_BA, corr_info_BA = phase_finder.find_theta_isi_hilbert( BA_tsp1_list, BA_tsp2_list) # Pair precession neuro_keys_dict1 = dict(tsp='tsp1', spikev='spike1v', spikex='spike1x', spikey='spike1y', spikeangle='spike1angle') neuro_keys_dict2 = dict(tsp='tsp2', spikev='spike2v', spikex='spike2x', spikey='spike2y', spikeangle='spike2angle') accept_mask = (~pairedpasses['rejected']) & ( pairedpasses['chunked'] < 2) & ( (pairedpasses['direction'] == 'A->B') | (pairedpasses['direction'] == 'B->A')) pairedpasses['excluded_for_precess'] = ~accept_mask precess_dfp = precesser.get_single_precession(pairedpasses, neuro_keys_dict1, field_d_union, tag='1') precess_dfp = precesser.get_single_precession(precess_dfp, neuro_keys_dict2, field_d_union, tag='2') precess_dfp = precess_filter.filter_pair(precess_dfp) fitted_precess_dfp = precess_dfp[ precess_dfp['fitted1'] & precess_dfp['fitted2']].reset_index(drop=True) # Paired spikes if (len(paired_tsp_list) == 0) or (len(passangles_all) == 0): continue hd_pair = np.concatenate(passangles_all) x_pair, y_pair = np.concatenate(x_all), np.concatenate(y_all) pos_pair = np.stack([x_pair, y_pair]).T paired_tsp = np.concatenate(paired_tsp_list) paired_tsp = paired_tsp[(paired_tsp <= all_maxt) & (paired_tsp >= all_mint)] if paired_tsp.shape[0] < 1: continue num_spikes_pair = paired_tsp.shape[0] hdsp_pair = interpolater_angle(paired_tsp) xsp_pair = interpolater_x(paired_tsp) ysp_pair = interpolater_y(paired_tsp) possp_pair = np.stack([xsp_pair, ysp_pair]).T aver_rate_pair = num_spikes_pair / (x_pair.shape[0] * dt) # Pair Directionality occbinsp, _ = np.histogram(hd_pair, bins=aedges) spbinsp, _ = np.histogram(hdsp_pair, bins=aedges) mlmer_pair = DirectionerMLM(pos_pair, hd_pair, dt, sp_binwidth, abind) rate_anglep, rate_Rp, normprobp_mlm = mlmer_pair.get_directionality( possp_pair, hdsp_pair) normprobp_mlm[np.isnan(normprobp_mlm)] = 0 # Time shift shuffling rate_R_pvalp = timeshift_shuffle_exp_wrapper( paired_tsp_list, t_all, rate_Rp, NShuffles, mlmer_pair, interpolater_x, interpolater_y, interpolater_angle, trange) # Rates with np.errstate(divide='ignore', invalid='ignore'): # None means no sample rate_AB = np.sum(nspikes_AB_list) / np.sum(duration_AB_list) rate_BA = np.sum(nspikes_BA_list) / np.sum(duration_BA_list) corate = np.sum(nspikes_AB_list + nspikes_BA_list) / np.sum(duration_AB_list + duration_BA_list) pair_rate = num_spikes_pair / np.sum(duration_AB_list + duration_BA_list) # KLD kld = calc_kld(normprob1_mlm, normprob2_mlm, normprobp_mlm) pairdata_dict['neuron1id'].append(i) pairdata_dict['neuron2id'].append(j) pairdata_dict['neuron1pos'].append(NeuronPos.iloc[i].to_numpy()) pairdata_dict['neuron2pos'].append(NeuronPos.iloc[j].to_numpy()) pairdata_dict['neurondist'].append(neurondist) pairdata_dict['border1'].append(border1) pairdata_dict['border2'].append(border2) pairdata_dict['aver_rate1'].append(aver_rate1) pairdata_dict['aver_rate2'].append(aver_rate2) pairdata_dict['aver_rate_pair'].append(aver_rate_pair) pairdata_dict['com1'].append(NeuronPos.iloc[i].to_numpy()) pairdata_dict['com2'].append(NeuronPos.iloc[j].to_numpy()) pairdata_dict['rate_angle1'].append(rate_angle1) pairdata_dict['rate_angle2'].append(rate_angle2) pairdata_dict['rate_anglep'].append(rate_anglep) pairdata_dict['rate_R1'].append(rate_R1) pairdata_dict['rate_R2'].append(rate_R2) pairdata_dict['rate_Rp'].append(rate_Rp) pairdata_dict['num_spikes1'].append(nspks1) pairdata_dict['num_spikes2'].append(nspks2) pairdata_dict['num_spikes_pair'].append(num_spikes_pair) pairdata_dict['phaselag_AB'].append(phaselag_AB) pairdata_dict['phaselag_BA'].append(phaselag_BA) pairdata_dict['corr_info_AB'].append(corr_info_AB) pairdata_dict['corr_info_BA'].append(corr_info_BA) pairdata_dict['rate_AB'].append(rate_AB) pairdata_dict['rate_BA'].append(rate_BA) pairdata_dict['corate'].append(corate) pairdata_dict['pair_rate'].append(pair_rate) pairdata_dict['kld'].append(kld) pairdata_dict['rate_R_pvalp'].append(rate_R_pvalp) pairdata_dict['precess_df1'].append(fitted_precessdf1) pairdata_dict['precess_angle1'].append(precessangle1) pairdata_dict['precess_R1'].append(precessR1) pairdata_dict['precess_df2'].append(fitted_precessdf2) pairdata_dict['precess_angle2'].append(precessangle2) pairdata_dict['precess_R2'].append(precessR2) pairdata_dict['numpass_at_precess1'].append(numpass_at_precess1) pairdata_dict['numpass_at_precess2'].append(numpass_at_precess2) pairdata_dict['precess_dfp'].append(fitted_precess_dfp) progress = progress + 1 pairdata = pd.DataFrame(pairdata_dict) # Convert distance to overlap dist_range = pairdata['neurondist'].max() - pairdata['neurondist'].min() pairdata['overlap'] = (pairdata['neurondist'].max() - pairdata['neurondist']) / dist_range pairdata.to_pickle(save_pth) pairdata = append_extrinsicity(pairdata) pairdata.to_pickle(save_pth) return pairdata
def single_field_preprocess_exp(df, vthresh=5, sthresh=3, NShuffles=200, save_pth=None): fielddf_dict = dict(ca=[], num_spikes=[], border=[], aver_rate=[], peak_rate=[], rate_angle=[], rate_R=[], rate_R_pval=[], minocc=[], field_area=[], field_bound=[], precess_df=[], precess_angle=[], precess_angle_low=[], precess_R=[], precess_R_pval=[], numpass_at_precess=[], numpass_at_precess_low=[]) num_trials = df.shape[0] aedges = np.linspace(-np.pi, np.pi, 36) abind = aedges[1] - aedges[0] sp_binwidth = 5 aedges_precess = np.linspace(-np.pi, np.pi, 6) kappa_precess = 1 precess_filter = PrecessionFilter() nspikes_stats = { 'CA1': 6, 'CA2': 6, 'CA3': 7 } # 25% quantile of precessing passes for ntrial in range(num_trials): wave = df.loc[ntrial, 'wave'] precesser = PrecessionProcesser(wave=wave) for ca in ['CA%d' % (i + 1) for i in range(3)]: # Get data field_df = df.loc[ntrial, ca + 'fields'] indata = df.loc[ntrial, ca + 'indata'] num_fields = field_df.shape[0] if indata.shape[0] < 1: continue tunner = IndataProcessor(indata, vthresh=vthresh, sthresh=sthresh, minpasstime=0.4) interpolater_angle = interp1d(tunner.t, tunner.angle) interpolater_x = interp1d(tunner.t, tunner.x) interpolater_y = interp1d(tunner.t, tunner.y) trange = (tunner.t.max(), tunner.t.min()) dt = tunner.t[1] - tunner.t[0] precesser.set_trange(trange) for nf in range(num_fields): print('%d/%d trial, %s, %d/%d field' % (ntrial, num_trials, ca, nf, num_fields)) # Get field info mask, pf, xyval = field_df.loc[nf, ['mask', 'pf', 'xyval']] tsp = field_df.loc[nf, 'xytsp']['tsp'] xaxis, yaxis = pf['X'][:, 0], pf['Y'][0, :] field_area = np.sum(mask) field_d = np.sqrt(field_area / np.pi) * 2 border = check_border(mask, margin=2) # Construct passes (segment & chunk) tok, idin = tunner.get_idin(mask, xaxis, yaxis) passdf = tunner.construct_singlefield_passdf( tok, tsp, interpolater_x, interpolater_y, interpolater_angle) allchunk_df = passdf[(~passdf['rejected']) & (passdf['chunked'] < 2)].reset_index( drop=True) # allchunk_df = passdf[(~passdf['rejected'])].reset_index(drop=True) # Get info from passdf and interpolate if allchunk_df.shape[0] < 1: continue all_x_list, all_y_list = allchunk_df['x'].to_list( ), allchunk_df['y'].to_list() all_t_list, all_passangles_list = allchunk_df['t'].to_list( ), allchunk_df['angle'].to_list() all_tsp_list, all_chunked_list = allchunk_df['tsp'].to_list( ), allchunk_df['chunked'].to_list() all_x = np.concatenate(all_x_list) all_y = np.concatenate(all_y_list) all_passangles = np.concatenate(all_passangles_list) all_tsp = np.concatenate(all_tsp_list) all_anglesp = np.concatenate( allchunk_df['spikeangle'].to_list()) xsp, ysp = np.concatenate( allchunk_df['spikex'].to_list()), np.concatenate( allchunk_df['spikey'].to_list()) pos = np.stack([all_x, all_y]).T possp = np.stack([xsp, ysp]).T # Average firing rate aver_rate = all_tsp.shape[0] / (all_x.shape[0] * dt) peak_rate = np.max(field_df.loc[nf, 'pf']['map'] * mask) # Field's directionality - need angle, anglesp, pos num_spikes = all_tsp.shape[0] occ_bins, _ = np.histogram(all_passangles, bins=aedges) minocc = occ_bins.min() mlmer = DirectionerMLM(pos, all_passangles, dt, sp_binwidth=sp_binwidth, a_binwidth=abind) rate_angle, rate_R, norm_prob_mlm = mlmer.get_directionality( possp, all_anglesp) # Time shift shuffling for rate directionality if np.isnan(rate_R): rate_R_pval = np.nan else: rate_R_pval = timeshift_shuffle_exp_wrapper( all_tsp_list, all_t_list, rate_R, NShuffles, mlmer, interpolater_x, interpolater_y, interpolater_angle, trange) # Precession per pass neuro_keys_dict = dict(tsp='tsp', spikev='spikev', spikex='spikex', spikey='spikey', spikeangle='spikeangle') accept_mask = (~passdf['rejected']) & (passdf['chunked'] < 2) passdf['excluded_for_precess'] = ~accept_mask precessdf, precess_angle, precess_R, _ = get_single_precessdf( passdf, precesser, precess_filter, neuro_keys_dict, field_d=field_d, kappa=kappa_precess, bins=None) fitted_precessdf = precessdf[precessdf['fitted']].reset_index( drop=True) # Proceed only if precession exists if (precess_angle is not None) and ( fitted_precessdf['precess_exist'].sum() > 0): # Post-hoc precession exclusion _, binR, postdoc_dens = compute_precessangle( pass_angles=fitted_precessdf['mean_anglesp'].to_numpy( ), pass_nspikes=fitted_precessdf['pass_nspikes'].to_numpy( ), precess_mask=fitted_precessdf['precess_exist']. to_numpy(), kappa=None, bins=aedges_precess) (_, passbins_p, passbins_np, _) = postdoc_dens all_passbins = passbins_p + passbins_np numpass_at_precess = get_numpass_at_angle( target_angle=precess_angle, aedge=aedges_precess, all_passbins=all_passbins) # Precession - low-spike passes ldf = fitted_precessdf[fitted_precessdf['pass_nspikes'] < nspikes_stats[ca]] # 25% quantile if (ldf.shape[0] > 0) and (ldf['precess_exist'].sum() > 0): precess_angle_low, _, _ = compute_precessangle( pass_angles=ldf['mean_anglesp'].to_numpy(), pass_nspikes=ldf['pass_nspikes'].to_numpy(), precess_mask=ldf['precess_exist'].to_numpy(), kappa=kappa_precess, bins=None) _, _, postdoc_dens_low = compute_precessangle( pass_angles=ldf['mean_anglesp'].to_numpy(), pass_nspikes=ldf['pass_nspikes'].to_numpy(), precess_mask=ldf['precess_exist'].to_numpy(), kappa=None, bins=aedges_precess) (_, passbins_p_low, passbins_np_low, _) = postdoc_dens_low all_passbins_low = passbins_p_low + passbins_np_low numpass_at_precess_low = get_numpass_at_angle( target_angle=precess_angle_low, aedge=aedges_precess, all_passbins=all_passbins_low) else: precess_angle_low = None numpass_at_precess_low = None # # Precession - time shift shuffle # psr = PassShuffler(all_t_list, all_tsp_list) # all_shuffled_R = np.zeros(NShuffles) # shufi = 0 # fail_times = 0 # repeated_times = 0 # tmpt = time.time() # while (shufi < NShuffles) & (fail_times < NShuffles): # # # Re-construct passdf # shifted_tsp_boxes = psr.timeshift_shuffle(seed=shufi+fail_times, return_concat=False) # # # shifted_spikex_boxes, shifted_spikey_boxes = [], [] # shifted_spikeangle_boxes, shifted_rejected_boxes = [], [] # # for boxidx, shuffled_tsp_box in enumerate(shifted_tsp_boxes): # shuffled_tsp_box = shuffled_tsp_box[(shuffled_tsp_box < trange[0]) & (shuffled_tsp_box > trange[1])] # shifted_spikex_boxes.append(interpolater_x(shuffled_tsp_box)) # shifted_spikey_boxes.append(interpolater_y(shuffled_tsp_box)) # shifted_spikeangle_boxes.append(interpolater_angle(shuffled_tsp_box)) # rejected, _ = tunner.rejection_singlefield(shuffled_tsp_box, all_t_list[boxidx], all_passangles_list[boxidx]) # shifted_rejected_boxes.append(rejected) # shuffled_passdf = pd.DataFrame({'x':all_x_list, 'y':all_y_list, 't':all_t_list, 'angle':all_passangles_list, # 'chunked':all_chunked_list, 'rejected':shifted_rejected_boxes, # neuro_keys_dict['tsp']:shifted_tsp_boxes, neuro_keys_dict['spikex']:shifted_spikex_boxes, # neuro_keys_dict['spikey']:shifted_spikey_boxes, neuro_keys_dict['spikeangle']:shifted_spikeangle_boxes}) # shuffled_accept_mask = (~shuffled_passdf['rejected']) & (shuffled_passdf['chunked'] < 2) # shuffled_passdf['excluded_for_precess'] = ~shuffled_accept_mask # shuf_precessdf, shuf_precessangle, shuf_precessR, _ = get_single_precessdf(shuffled_passdf, precesser, precess_filter, neuro_keys_dict, occ_bins.min(), # field_d=field_d, kappa=kappa_precess, bins=None) # # if (shuf_precessdf.shape[0] > 0) and (shuf_precessR is not None): # all_shuffled_R[shufi] = shuf_precessR # shufi += 1 # else: # fail_times += 1 # precess_R_pval = 1 - np.nanmean(precess_R > all_shuffled_R) # precess_shuffletime = time.time()-tmpt # print('Shuf %0.4f - %0.4f - %0.4f, Target %0.4f'%(np.quantile(all_shuffled_R, 0.25), np.quantile(all_shuffled_R, 0.5), np.quantile(all_shuffled_R, 0.75), precess_R)) # print('Best precess=%0.2f, pval=%0.5f, time=%0.2f, failed=%d, repeated=%d'%(precess_angle, precess_R_pval, precess_shuffletime, fail_times, repeated_times)) precess_R_pval = 1 else: numpass_at_precess = None precess_angle_low = None numpass_at_precess_low = None precess_R_pval = None fielddf_dict['ca'].append(ca) fielddf_dict['num_spikes'].append(num_spikes) fielddf_dict['field_area'].append(field_area) fielddf_dict['field_bound'].append(xyval) fielddf_dict['border'].append(border) fielddf_dict['aver_rate'].append(aver_rate) fielddf_dict['peak_rate'].append(peak_rate) fielddf_dict['rate_angle'].append(rate_angle) fielddf_dict['rate_R'].append(rate_R) fielddf_dict['rate_R_pval'].append(rate_R_pval) fielddf_dict['minocc'].append(minocc) fielddf_dict['precess_df'].append(fitted_precessdf) fielddf_dict['precess_angle'].append(precess_angle) fielddf_dict['precess_angle_low'].append(precess_angle_low) fielddf_dict['precess_R'].append(precess_R) fielddf_dict['precess_R_pval'].append(precess_R_pval) fielddf_dict['numpass_at_precess'].append(numpass_at_precess) fielddf_dict['numpass_at_precess_low'].append( numpass_at_precess_low) # tmpdf = pd.DataFrame(dict(ca=fielddf_dict['ca'], pval=fielddf_dict['precess_R_pval'])) # for catmp, cadftmp in tmpdf.groupby('ca'): # nonnan_count = cadftmp[~cadftmp['pval'].isna()].shape[0] # if nonnan_count ==0: # nonnan_count = 1 # sig_count = cadftmp[cadftmp['pval'] < 0.05].shape[0] # print('%s: ALL %d/%d=%0.2f, Among Precess %d/%d=%0.2f'%(catmp, sig_count, cadftmp.shape[0], # sig_count/cadftmp.shape[0], sig_count, # nonnan_count, sig_count/nonnan_count)) fielddf_raw = pd.DataFrame(fielddf_dict) # Assign field ids within ca fielddf_raw['fieldid_ca'] = 0 for ca, cadf in fielddf_raw.groupby('ca'): fieldidca = np.arange(cadf.shape[0]) + 1 index_ca = cadf.index fielddf_raw.loc[index_ca, 'fieldid_ca'] = fieldidca fielddf_raw.to_pickle(save_pth) return fielddf_raw
def single_field_preprocess_Romani(simdata, radius=2, vthresh=2, sthresh=80, NShuffles=200, save_pth=None): """ Parameters ---------- Indata SpikeData NeuronPos radius vthresh : float Default = 2. Determined by the ratio between avergae speed in Emily's data and the target vthresh 5 there. sthresh : float Default = 80. Determined by the same percentile (10%) of passes excluded in Emily's data (sthresh = 3 there). It is different since straightrank depends on sampling frequency. It is 1ms in sumulation. subsample_fraction : float The fraction that the spikes would be subsampled. Returns ------- """ subsample_fraction = 0.4 datadict = dict(num_spikes=[], border=[], aver_rate=[], rate_angle=[], rate_R=[], rate_R_pval=[], minocc=[], precess_df=[], precess_angle=[], precess_angle_low=[], precess_R=[], numpass_at_precess=[], numpass_at_precess_low=[]) Indata, SpikeData, NeuronPos = simdata['Indata'], simdata[ 'SpikeData'], simdata['NeuronPos'] aedges = np.linspace(-np.pi, np.pi, 36) abind = aedges[1] - aedges[0] sp_binwidth = 2 * np.pi / 16 tunner = IndataProcessor(Indata, vthresh=vthresh, sthresh=sthresh, minpasstime=0.4) wave = dict(tax=Indata['t'].to_numpy(), phase=Indata['phase'].to_numpy(), theta=np.ones(Indata.shape[0])) interpolater_angle = interp1d(tunner.t, tunner.angle) interpolater_x = interp1d(tunner.t, tunner.x) interpolater_y = interp1d(tunner.t, tunner.y) trange = (tunner.t.max(), tunner.t.min()) dt = tunner.t[1] - tunner.t[0] precesser = PrecessionProcesser(wave=wave) precesser.set_trange(trange) precess_filter = PrecessionFilter() lowspike_num = 13 kappa_precess = 1 aedges_precess = np.linspace(-np.pi, np.pi, 6) num_neurons = NeuronPos.shape[0] for nidx in range(num_neurons): print('%d/%d Neuron' % (nidx, num_neurons)) # Get spike indexes + subsample spdf = SpikeData[SpikeData['neuronidx'] == nidx].reset_index(drop=True) tidxsp = spdf['tidxsp'].to_numpy().astype(int) np.random.seed(nidx) sampled_tidxsp = np.random.choice(tidxsp.shape[0] - 1, int(tidxsp.shape[0] * subsample_fraction), replace=False) sampled_tidxsp.sort() tidxsp = tidxsp[sampled_tidxsp] tsp = Indata.loc[tidxsp, 't'].to_numpy() neuron_pos = NeuronPos.iloc[nidx].to_numpy() # Check border border = check_border_sim(neuron_pos[0], neuron_pos[1], radius, (0, 2 * np.pi)) # Construct passdf dist = np.sqrt((neuron_pos[0] - tunner.x)**2 + (neuron_pos[1] - tunner.y)**2) tok = dist < radius passdf = tunner.construct_singlefield_passdf(tok, tsp, interpolater_x, interpolater_y, interpolater_angle) # allchunk_df = passdf[(~passdf['rejected']) & (passdf['chunked']<2)].reset_index(drop=True) allchunk_df = passdf[(~passdf['rejected'])].reset_index(drop=True) # Get info from passdf and interpolate if allchunk_df.shape[0] < 1: continue all_x_list, all_y_list = allchunk_df['x'].to_list( ), allchunk_df['y'].to_list() all_t_list, all_passangles_list = allchunk_df['t'].to_list( ), allchunk_df['angle'].to_list() all_tsp_list, all_chunked_list = allchunk_df['tsp'].to_list( ), allchunk_df['chunked'].to_list() all_x = np.concatenate(all_x_list) all_y = np.concatenate(all_y_list) all_passangles = np.concatenate(all_passangles_list) all_tsp = np.concatenate(all_tsp_list) all_anglesp = np.concatenate(allchunk_df['spikeangle'].to_list()) xsp, ysp = np.concatenate( allchunk_df['spikex'].to_list()), np.concatenate( allchunk_df['spikey'].to_list()) pos = np.stack([all_x, all_y]).T possp = np.stack([xsp, ysp]).T # Average firing rate aver_rate = all_tsp.shape[0] / (all_x.shape[0] * dt) # Field's directionality num_spikes = all_tsp.shape[0] occ_bins, _ = np.histogram(all_passangles, bins=aedges) minocc = occ_bins.min() mlmer = DirectionerMLM(pos, all_passangles, dt, sp_binwidth=sp_binwidth, a_binwidth=abind) rate_angle, rate_R, norm_prob_mlm = mlmer.get_directionality( possp, all_anglesp) # Time shift shuffling for rate directionality rate_R_pval = timeshift_shuffle_exp_wrapper(all_tsp_list, all_t_list, rate_R, NShuffles, mlmer, interpolater_x, interpolater_y, interpolater_angle, trange) # Precession per pass neuro_keys_dict = dict(tsp='tsp', spikev='spikev', spikex='spikex', spikey='spikey', spikeangle='spikeangle') accept_mask = (~passdf['rejected']) & (passdf['chunked'] < 2) passdf['excluded_for_precess'] = ~accept_mask precessdf, precess_angle, precess_R, _ = get_single_precessdf( passdf, precesser, precess_filter, neuro_keys_dict, field_d=radius * 2, kappa=kappa_precess, bins=None) fitted_precessdf = precessdf[precessdf['fitted']].reset_index( drop=True) # Proceed only if precession exists if (precess_angle is not None) and (fitted_precessdf['precess_exist'].sum() > 0): # Post-hoc precession exclusion _, binR, postdoc_dens = compute_precessangle( pass_angles=fitted_precessdf['mean_anglesp'].to_numpy(), pass_nspikes=fitted_precessdf['pass_nspikes'].to_numpy(), precess_mask=fitted_precessdf['precess_exist'].to_numpy(), kappa=None, bins=aedges_precess) (_, passbins_p, passbins_np, _) = postdoc_dens all_passbins = passbins_p + passbins_np numpass_at_precess = get_numpass_at_angle( target_angle=precess_angle, aedge=aedges_precess, all_passbins=all_passbins) # Precession - low-spike passes ldf = fitted_precessdf[fitted_precessdf['pass_nspikes'] < lowspike_num] # 25% quantile if (ldf.shape[0] > 0) and (ldf['precess_exist'].sum() > 0): precess_angle_low, _, _ = compute_precessangle( pass_angles=ldf['mean_anglesp'].to_numpy(), pass_nspikes=ldf['pass_nspikes'].to_numpy(), precess_mask=ldf['precess_exist'].to_numpy(), kappa=kappa_precess, bins=None) _, _, postdoc_dens_low = compute_precessangle( pass_angles=ldf['mean_anglesp'].to_numpy(), pass_nspikes=ldf['pass_nspikes'].to_numpy(), precess_mask=ldf['precess_exist'].to_numpy(), kappa=None, bins=aedges_precess) (_, passbins_p_low, passbins_np_low, _) = postdoc_dens_low all_passbins_low = passbins_p_low + passbins_np_low numpass_at_precess_low = get_numpass_at_angle( target_angle=precess_angle_low, aedge=aedges_precess, all_passbins=all_passbins_low) else: precess_angle_low = None numpass_at_precess_low = None else: numpass_at_precess = None precess_angle_low = None numpass_at_precess_low = None datadict['num_spikes'].append(num_spikes) datadict['border'].append(border) datadict['aver_rate'].append(aver_rate) # datadict['rate_angle'].append(rate_angle) datadict['rate_R'].append(rate_R) datadict['rate_R_pval'].append(rate_R_pval) datadict['minocc'].append(minocc) datadict['precess_df'].append(precessdf) datadict['precess_angle'].append(precess_angle) datadict['precess_angle_low'].append(precess_angle_low) datadict['precess_R'].append(precess_R) datadict['numpass_at_precess'].append(numpass_at_precess) datadict['numpass_at_precess_low'].append(numpass_at_precess_low) datadf = pd.DataFrame(datadict) datadf.to_pickle(save_pth)