def nway_analysis(full_raster, meta):
# trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
trialR, _, _ = cdPCA.format_raster(full_raster)
trialR = trialR.squeeze() # squeezes out probe
R, C, S, T = trialR.shape
# calculates full LDA. i.e. considering all 4 categories
LDA_projection, LDA_transformation = cLDA.fit_transform_over_time(
trialR, 1)
dprime = cDP.pairwise_dprimes(LDA_projection.squeeze())
# calculates floor (ctx shuffle) and ceiling (simulated data)
sim_dprime = np.empty([meta['montecarlo']] + list(dprime.shape))
shuf_dprime = np.empty([meta['montecarlo']] + list(dprime.shape))
ctx_shuffle = trialR.copy()
pbar = ProgressBar()
for rr in pbar(range(meta['montecarlo'])):
# ceiling: simulates data, calculates dprimes
sim_trial = np.random.normal(np.mean(trialR, axis=0),
np.std(trialR, axis=0),
size=[R, C, S, T])
sim_projection = cLDA.transform_over_time(
cLDA._reorder_dims(sim_trial), LDA_transformation)
sim_dprime[rr, ...] = cDP.pairwise_dprimes(
cLDA._recover_dims(sim_projection).squeeze())
ctx_shuffle = shuffle(ctx_shuffle, shuffle_axis=2, indie_axis=0)
shuf_projection, _ = cLDA.fit_transform_over_time(ctx_shuffle)
shuf_dprime[rr, ...] = cDP.pairwise_dprimes(shuf_projection.squeeze())
return dprime, shuf_dprime, sim_dprime
def dPCA_fourway_analysis(site, probe, meta):
# recs = load(site, remote=True, rasterfs=meta['raster_fs'], recache=False)
recs = load(site, rasterfs=meta['raster_fs'], recache=rec_recache)
if len(recs) > 2:
print(f'\n\n{recs.keys()}\n\n')
rec = recs['trip0']
sig = rec['resp']
# calculates response realiability and select only good cells to improve analysis
r_vals, goodcells = signal_reliability(sig,
r'\ASTIM_*',
threshold=meta['reliability'])
goodcells = goodcells.tolist()
# get the full data raster Context x Probe x Rep x Neuron x Time
raster = src.data.rasters.raster_from_sig(
sig,
probe,
channels=goodcells,
contexts=meta['transitions'],
smooth_window=meta['smoothing_window'],
raster_fs=meta['raster_fs'],
zscore=meta['zscore'])
# trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
trialR, R, _ = cdPCA.format_raster(raster)
trialR, R = trialR.squeeze(axis=3), R.squeeze(axis=2) # squeezes out probe
Re, C, S, T = trialR.shape
# calculates full dPCA. i.e. considering all 4 categories
dPCA_projection, dPCA_transformation = cdPCA.fit_transform(R, trialR)
dprime = cDP.pairwise_dprimes(dPCA_projection,
observation_axis=0,
condition_axis=1)
# calculates floor (ctx shuffle) and ceiling (simulated data)
sim_dprime = np.empty([meta['montecarlo']] + list(dprime.shape))
shuf_dprime = np.empty([meta['montecarlo']] + list(dprime.shape))
ctx_shuffle = trialR.copy()
# pbar = ProgressBar()
for rr in range(meta['montecarlo']):
# ceiling: simulates data, calculates dprimes
sim_trial = np.random.normal(np.mean(trialR, axis=0),
np.std(trialR, axis=0),
size=[Re, C, S, T])
sim_projection = cdPCA.transform(sim_trial, dPCA_transformation)
sim_dprime[rr, ...] = cDP.pairwise_dprimes(sim_projection,
observation_axis=0,
condition_axis=1)
ctx_shuffle = shuffle(ctx_shuffle, shuffle_axis=2, indie_axis=0)
shuf_projection = cdPCA.transform(ctx_shuffle, dPCA_transformation)
shuf_dprime[rr, ...] = cDP.pairwise_dprimes(shuf_projection,
observation_axis=0,
condition_axis=1)
return dprime, shuf_dprime, sim_dprime, goodcells
def full_dPCA_dprimes(site, contexts, probes, meta):
raster, goodcells = _load_site_formated_raste(site, contexts, probes, meta)
# trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
trialR, R, _ = cdPCA.format_raster(raster)
# calculates full dPCA. i.e. considering all 4 categories
_, trialZ, dpca = cdPCA._cpp_dPCA(R, trialR)
dPCA_projection = trialZ['ct'][:, 0, ...]
dprime = cDP.pairwise_dprimes(dPCA_projection,
observation_axis=0,
condition_axis=1,
flip=meta['dprime_absolute'])
# calculates the variance explained. special case for full dpca, not present in other dprime approaches
var_capt = cdPCA.variance_captured(dpca, R)
# Shuffles the rasters n times and organizes in an array with the same shape the raster plus one dimension
# with size n containing each shuffle
# calculates the pairwise dprime
rep, ctx, prb, tme = dPCA_projection.shape
transition_pairs = list(itt.combinations(contexts, 2))
print(f"\nshuffling {meta['montecarlo']} times")
rng = np.random.default_rng(42)
shuf_projections = np.empty([meta['montecarlo'], rep, ctx, prb, tme])
shuf_trialR = trialR.copy()
for rr in range(meta['montecarlo']):
shuf_trialR = shuffle(shuf_trialR,
shuffle_axis=2,
indie_axis=0,
rng=rng)
shuf_R = np.mean(shuf_trialR, axis=0)
# saves the first regularizer to speed things up.
if rr == 0:
_, shuf_trialZ, shuf_dpca = cdPCA._cpp_dPCA(shuf_R, shuf_trialR)
regularizer = shuf_dpca.regularizer
else:
_, shuf_trialZ, dpca = cdPCA._cpp_dPCA(
shuf_R, shuf_trialR, {'regularizer': regularizer})
shuf_projections[rr, ...] = shuf_trialZ['ct'][:, 0, ...]
shuffled_dprime = cDP.pairwise_dprimes(shuf_projections,
observation_axis=1,
condition_axis=2,
flip=meta['dprime_absolute'])
# add dimension for single PC, for compatibility with single cell arrays
dprime = np.expand_dims(dprime, axis=0)
shuffled_dprime = np.expand_dims(shuffled_dprime, axis=1)
return dprime, shuffled_dprime, goodcells, var_capt
def single_cell_dprimes(site, contexts, probes, meta):
"""
calculated the dprime between context for all probes and for all cells in a site. Calculates significance using
montecarlo shuffling of the context identity
:param site: string identifying the site
:param probes: list of probe numbers
:param meta: dict with meta parameters
:return: dprime (ndarray with shape Unit x Ctx_pair x Probe x Time),
shuffled_dprimes (ndarray with shape Montecarlo x Unit x Ctx_pair x Probe x Time),
goocells (list of strings)
"""
raster, goodcells = _load_site_formated_raste(site, contexts, probes, meta)
# trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
trialR, _, _ = cdPCA.format_raster(raster)
rep, chn, ctx, prb, tme = trialR.shape
transition_pairs = list(itt.combinations(contexts, 2))
dprime = cDP.pairwise_dprimes(
trialR,
observation_axis=0,
condition_axis=2,
flip=meta['dprime_absolute']) # shape Cell x CtxPair x Probe x Time
# Shuffles the rasters n times and organizes in an array with the same shape the raster plus one dimension
# with size n containing each shuffle, then calculates the pairwise dprime
print(f"\nshuffling {meta['montecarlo']} times")
rng = np.random.default_rng(42)
shuf_trialR = np.empty((meta['montecarlo'], rep, chn, ctx, prb, tme))
ctx_shuffle = trialR.copy()
for rr in range(meta['montecarlo']):
shuf_trialR[rr, ...] = shuffle(ctx_shuffle,
shuffle_axis=2,
indie_axis=0,
rng=rng)
shuffled_dprime = cDP.pairwise_dprimes(shuf_trialR,
observation_axis=1,
condition_axis=3,
flip=meta['dprime_absolute'])
return dprime, shuffled_dprime, goodcells, None
def dPCA_fourway_analysis(site, probe, meta):
recs = load(site)
if len(recs) > 2:
print(f'\n\n{recs.keys()}\n\n')
rec = recs['trip0']
sig = rec['resp']
# calculates response realiability and select only good cells to improve analysis
r_vals, goodcells = signal_reliability(sig,
r'\ASTIM_*',
threshold=meta['reliability'])
goodcells = goodcells.tolist()
# get the full data raster Context x Probe x Rep x Neuron x Time
raster = src.data.rasters.raster_from_sig(
sig,
probe,
channels=goodcells,
contexts=meta['transitions'],
smooth_window=meta['smoothing_window'],
raster_fs=meta['raster_fs'],
zscore=meta['zscore'])
# trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
trialR, R, _ = cdPCA.format_raster(raster)
trialR, R = trialR.squeeze(), R.squeeze() # squeezes out probe
Re, C, S, T = trialR.shape
# calculates full dPCA. i.e. considering all 4 categories
def fit_transformt(R, trialR):
_, dPCA_projection, _, dpca = cdPCA._cpp_dPCA(R,
trialR,
significance=False,
dPCA_parms={})
dPCA_projection = dPCA_projection['ct'][:, 0, ]
dPCA_transformation = np.tile(dpca.D['ct'][:, 0][:, None, None],
[1, 1, T])
return dPCA_projection, dPCA_transformation
dPCA_projection, dPCA_transformation = fit_transformt(R, trialR)
dprime = cDP.pairwise_dprimes(dPCA_projection)
# calculates floor (ctx shuffle) and ceiling (simulated data)
sim_dprime = np.empty([meta['montecarlo']] + list(dprime.shape))
shuf_dprime = np.empty([meta['montecarlo']] + list(dprime.shape))
ctx_shuffle = trialR.copy()
pbar = ProgressBar()
for rr in pbar(range(meta['montecarlo'])):
# ceiling: simulates data, calculates dprimes
sim_trial = np.random.normal(np.mean(trialR, axis=0),
np.std(trialR, axis=0),
size=[Re, C, S, T])
sim_projection = cLDA.transform_over_time(
cLDA._reorder_dims(sim_trial), dPCA_transformation)
sim_dprime[rr, ...] = cDP.pairwise_dprimes(
cLDA._recover_dims(sim_projection).squeeze())
ctx_shuffle = shuffle(ctx_shuffle, shuffle_axis=2, indie_axis=0)
shuf_projection = cLDA.transform_over_time(
cLDA._reorder_dims(ctx_shuffle), dPCA_transformation)
shuf_dprime[rr, ...] = cDP.pairwise_dprimes(
cLDA._recover_dims(shuf_projection).squeeze())
return dprime, shuf_dprime, sim_dprime
def probewise_LDA_dprimes(site, contexts, probes, meta):
"""
performs dimensionality reduction with LDA done independently for each probe. the uses the discriminated projection
to calculate the dprime between context for all probes in the site. Calculates
significance using montecarlo shuffling of the context identity.
:param site: string identifying the site
:param probes: list of probe numbers
:param meta: dict with meta parameters
:return: dprime (ndarray with shape Ctx_pair x Probe x Time),
shuffled_dprimes (ndarray with shape Montecarlo x Ctx_pair x Probe x Time),
goocells (list of strings)
"""
raster, goodcells = _load_site_formated_raste(site, contexts, probes, meta)
# trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
trialR, R, _ = cdPCA.format_raster(raster)
rep, unt, ctx, prb, tme = trialR.shape
transition_pairs = list(itt.combinations(contexts, 2))
# iterates over each probe
dprime = np.empty([len(transition_pairs), prb, tme])
shuffled_dprime = np.empty(
[meta['montecarlo'],
len(transition_pairs), prb, tme])
for pp in probes:
probe_idx = probes.index(pp)
probe_trialR = trialR[..., probe_idx, :]
# calculates LDA considering all 4 categories
LDA_projection, _ = cLDA.fit_transform_over_time(probe_trialR)
LDA_projection = LDA_projection.squeeze(
axis=1) # shape Trial x Context x Time
dprime[:, probe_idx, :] = cDP.pairwise_dprimes(
LDA_projection,
observation_axis=0,
condition_axis=1,
flip=meta['dprime_absolute'])
# Shuffles the rasters n times and organizes in an array with the same shape as the original raster plus one
# dimension with size meta['montecarlo'] containing each shuffle
# calculates the pairwise dprime
print(f"\nshuffling {meta['montecarlo']} times")
rng = np.random.default_rng(42)
shuf_projections = np.empty([meta['montecarlo'], rep, ctx, tme])
shuf_projections[:] = np.nan
shuf_trialR = probe_trialR.copy()
for rr in range(meta['montecarlo']):
shuf_trialR = shuffle(shuf_trialR,
shuffle_axis=2,
indie_axis=0,
rng=rng)
shuf_LDA_projection, _ = cLDA.fit_transform_over_time(shuf_trialR)
shuf_projections[rr, ...] = shuf_LDA_projection.squeeze(
axis=1) # shape Trial x Context x Time
shuffled_dprime[:, :, probe_idx, :] = cDP.pairwise_dprimes(
shuf_projections,
observation_axis=1,
condition_axis=2,
flip=meta['dprime_absolute'])
# add dimension for single PC, for compatibility with single cell arrays
dprime = np.expand_dims(dprime, axis=0)
shuffled_dprime = np.expand_dims(shuffled_dprime, axis=1)
return dprime, shuffled_dprime, goodcells, None
def probewise_dPCA_dprimes(site, contexts, probes, meta):
"""
performs dimensionality reduction with dPCA done independently for each probe. Then uses the first context dependent
demixed principal component to calculated the dprime between context for all probes in the site. Calculates
significance using montecarlo shuffling of the context identity.
:param site: string identifying the site
:param probes: list of probe numbers
:param meta: dict with meta parameters
:return: dprime (ndarray with shape PC x Ctx_pair x Probe x Time),
shuffled_dprimes (ndarray with shape Montecarlo x PC x Ctx_pair x Probe x Time),
goocells (list of strings)
"""
raster, goodcells = _load_site_formated_raste(site, contexts, probes, meta)
# trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
trialR, R, _ = cdPCA.format_raster(raster)
rep, unt, ctx, prb, tme = trialR.shape
transition_pairs = list(itt.combinations(contexts, 2))
# iterates over each probe
dprime = np.empty([len(transition_pairs), prb, tme])
shuffled_dprime = np.empty(
[meta['montecarlo'],
len(transition_pairs), prb, tme])
var_capt = list()
for pp in probes:
probe_idx = probes.index(pp)
probe_trialR = trialR[..., probe_idx, :]
probe_R = R[..., probe_idx, :]
# calculates dPCA considering all 4 categories
_, trialZ, dpca = cdPCA._cpp_dPCA(probe_R, probe_trialR)
var_capt.append(cdPCA.variance_captured(dpca, probe_R))
dPCA_projection = trialZ['ct'][:, 0, ...]
dprime[:, probe_idx, :] = cDP.pairwise_dprimes(
dPCA_projection,
observation_axis=0,
condition_axis=1,
flip=meta['dprime_absolute'])
# Shuffles the rasters n times and organizes in an array with the same shape as the original raster plus one
# dimension with size meta['montecarlo'] containing each shuffle
# calculates the pairwise dprime
print(f"\nshuffling {meta['montecarlo']} times")
rng = np.random.default_rng(42)
shuf_projections = np.empty([meta['montecarlo'], rep, ctx, tme])
shuf_projections[:] = np.nan
shuf_trialR = probe_trialR.copy()
for rr in range(meta['montecarlo']):
shuf_trialR = shuffle(shuf_trialR,
shuffle_axis=2,
indie_axis=0,
rng=rng)
shuf_R = np.mean(shuf_trialR, axis=0)
#saves the first regularizer to speed things up.
if rr == 0:
_, shuf_trialZ, shuf_dpca = cdPCA._cpp_dPCA(
shuf_R, shuf_trialR)
regularizer = shuf_dpca.regularizer
else:
_, shuf_trialZ, dpca = cdPCA._cpp_dPCA(
shuf_R, shuf_trialR, {'regularizer': regularizer})
shuf_projections[rr, :] = shuf_trialZ['ct'][:, 0, ...]
shuffled_dprime[:, :, probe_idx, :] = cDP.pairwise_dprimes(
shuf_projections,
observation_axis=1,
condition_axis=2,
flip=meta['dprime_absolute'])
# add dimension for single PC, for compatibility with single cell arrays
dprime = np.expand_dims(dprime, axis=0)
shuffled_dprime = np.expand_dims(shuffled_dprime, axis=1)
return dprime, shuffled_dprime, goodcells, var_capt
ceil_d = np.empty((nreps, dprime.shape[0]))
ctx_shuffle = trialR.copy()
for ii in range(nreps):
# dprime celing: normal simulation of the data, projection and dprime calculation
trial_sim = np.random.normal(np.mean(trialR, axis=0),
np.std(trialR, axis=0),
size=[R, C, S, T]).squeeze()
sim_proj = cLDA._recover_dims(
cLDA.transform_over_time(cLDA._reorder_dims(trial_sim), lda_axes))
ceil_d[ii, :] = cpd.dprime(sim_proj[:, :, 0, :].squeeze(),
sim_proj[:, :, 1, :].squeeze())
# dprime floor: shuffle context(dim2) identity by trial(dim0), projection and dprime calculation
ctx_shuffle = shuffle(ctx_shuffle, shuffle_axis=2, indie_axis=0)
shuf_proj, shuf_lda_axes = cLDA.fit_transform_over_time(ctx_shuffle, 1)
floor_d[ii, :] = cpd.dprime(shuf_proj[:, :, 0, :].squeeze(),
shuf_proj[:, :, 1, :].squeeze())
# # plots the floor and ceiling confidence intervals
# fig, ax = plt.subplots()
# # floor
# ax.plot(np.mean(floor_d, axis=0), color='red', alpha=1, label='floor')
# _cint(floor_d.T, 0.95, ax=ax, fillkwargs={'alpha': 0.5, 'color':'red'})
# # ceiling
# ax.plot(np.mean(ceil_d, axis=0), color='green', alpha=1, label='ceiling')
# _cint(ceil_d.T, 0.95, ax=ax, fillkwargs={'alpha': 0.5, 'color':'green'})
# # real dprime
# ax.plot(dprime, color='black', label='d prime')
# ax.legend()
def twoway_analysis(site, probe, meta):
recs = load(site)
if len(recs) > 2:
print(f'\n\n{recs.keys()}\n\n')
rec = recs['trip0']
sig = rec['resp']
# calculates response realiability and select only good cells to improve analysis
r_vals, goodcells = signal_reliability(sig,
r'\ASTIM_*',
threshold=meta['reliability'])
goodcells = goodcells.tolist()
# outer lists to save the dprimes foe each pair of ctxs
dprime = list()
shuf_dprime = list()
sim_dprime = list()
for transitions in itt.combinations(meta['transitions'], 2):
# get the full data raster Context x Probe x Rep x Neuron x Time
raster = src.data.rasters.raster_from_sig(
sig,
probe,
channels=goodcells,
contexts=transitions,
smooth_window=meta['smoothing_window'],
raster_fs=meta['raster_fs'],
zscore=meta['zscore'])
# trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
trialR, _, _ = cdPCA.format_raster(raster)
trialR = trialR.squeeze() # squeezes out probe
R, C, S, T = trialR.shape
# calculates LDA across the two selected transitions categories
LDA_projection, LDA_transformation = cLDA.fit_transform_over_time(
trialR, 1)
dp = cDP.pairwise_dprimes(LDA_projection.squeeze())
dprime.append(dp)
# calculates floor (ctx shuffle) and ceiling (simulated data)
sim_dp = np.empty([meta['montecarlo']] + list(dp.shape))
shuf_dp = np.empty([meta['montecarlo']] + list(dp.shape))
ctx_shuffle = trialR.copy()
pbar = ProgressBar()
for rr in pbar(range(meta['montecarlo'])):
# ceiling: simulates data, calculates dprimes
sim_trial = np.random.normal(np.mean(trialR, axis=0),
np.std(trialR, axis=0),
size=[R, C, S, T])
sim_projection = cLDA.transform_over_time(
cLDA._reorder_dims(sim_trial), LDA_transformation)
sim_dp[rr, ...] = cDP.pairwise_dprimes(
cLDA._recover_dims(sim_projection).squeeze())
ctx_shuffle = shuffle(ctx_shuffle, shuffle_axis=2, indie_axis=0)
shuf_projection, _ = cLDA.fit_transform_over_time(ctx_shuffle)
shuf_dp[rr, ...] = cDP.pairwise_dprimes(shuf_projection.squeeze())
shuf_dprime.append(shuf_dp)
sim_dprime.append(sim_dp)
# orders the list into arrays of the same shape as the fourwise analysis: MonteCarlo x Pair x Time
dprime = np.concatenate(dprime, axis=0)
shuf_dprime = np.concatenate(shuf_dprime, axis=1)
sim_dprime = np.concatenate(sim_dprime, axis=1)
return dprime, shuf_dprime, sim_dprime
def cell_dprime(site, probe, meta):
# recs = load(site, remote=True, rasterfs=meta['raster_fs'], recache=False)
recs = load(site, rasterfs=meta['raster_fs'], recache=rec_recache)
if len(recs) > 2:
print(f'\n\n{recs.keys()}\n\n')
rec = recs['trip0']
sig = rec['resp']
# calculates response realiability and select only good cells to improve analysis
r_vals, goodcells = signal_reliability(sig,
r'\ASTIM_*',
threshold=meta['reliability'])
goodcells = goodcells.tolist()
# get the full data raster Context x Probe x Rep x Neuron x Time
raster = src.data.rasters.raster_from_sig(
sig,
probe,
channels=goodcells,
contexts=meta['transitions'],
smooth_window=meta['smoothing_window'],
raster_fs=meta['raster_fs'],
zscore=meta['zscore'],
part='probe')
# trialR shape: Trial x Cell x Context x Probe x Time; R shape: Cell x Context x Probe x Time
trialR, R, _ = cdPCA.format_raster(raster)
trialR, R = trialR.squeeze(axis=3), R.squeeze(axis=2) # squeezes out probe
rep, chn, ctx, tme = trialR.shape
trans_pairs = [
f'{x}_{y}' for x, y in itt.combinations(meta['transitions'], 2)
]
dprime = cDP.pairwise_dprimes(
trialR, observation_axis=0,
condition_axis=2) # shape CellPair x Cell x Time
# Shuffles the rasters n times and organizes in an array with the same shape the raster plus one dimension
# with size n containing each shuffle
shuffled = list()
# pbar = ProgressBar()
print(f"\nshuffling {meta['montecarlo']} times")
for tp in trans_pairs:
shuf_trialR = np.empty([meta['montecarlo'], rep, chn, 2, tme])
shuf_trialR[:] = np.nan
tran_idx = np.array(
[meta['transitions'].index(t) for t in tp.split('_')])
ctx_shuffle = trialR[:, :, tran_idx, :].copy()
for rr in range(meta['montecarlo']):
shuf_trialR[rr, ...] = shuffle(ctx_shuffle,
shuffle_axis=2,
indie_axis=0)
shuffled.append(
cDP.pairwise_dprimes(shuf_trialR,
observation_axis=1,
condition_axis=3))
shuffled = np.stack(shuffled, axis=1).squeeze(axis=0).swapaxes(
0, 1) # shape Montecarlo x ContextPair x Cell x Time
return dprime, shuffled, goodcells, trans_pairs