def plot_2D_bytarget(dataDB, h5fname, dfSummary, trgChName, dropChannels=None):
pidTypes = ['unique', 'syn', 'red']
labels = dataDB.get_channel_labels()
trgIdx = labels.index(trgChName)
groupLst = sorted(list(set(dfSummary.columns) - {'key', 'mousename'}))
for key, dataMouse in dfSummary.groupby(groupLst):
mice = list(sorted(set(dataMouse['mousename'])))
dfJointDict = read_parse_joint_dataframe(dataMouse,
h5fname,
mice,
pidTypes,
dropChannels=dropChannels)
fig, ax = plt.subplots(ncols=3, figsize=(12, 4))
for iPid, pidType in enumerate(pidTypes):
Mrez3D = _bitdict_to_3Dmat(dataDB, dfJointDict, pidType, mice)
nChannel = Mrez3D.shape[0]
Mrez2D = Mrez3D[:, :, trgIdx]
print(np.max(Mrez2D))
imshow(fig,
ax[iPid],
Mrez2D,
title=pidType,
haveColorBar=True,
cmap='jet')
fig.suptitle('_'.join(key))
plt.show()
def plot_2D_avg(dataDB, h5fname, dfSummary, dropChannels=None, avgAxis=2):
pidTypes = ['unique', 'syn', 'red']
groupLst = sorted(list(set(dfSummary.columns) - {'key', 'mousename'}))
for key, dataMouse in dfSummary.groupby(groupLst):
mice = list(sorted(set(dataMouse['mousename'])))
dfJointDict = read_parse_joint_dataframe(dataMouse,
h5fname,
mice,
pidTypes,
dropChannels=dropChannels)
fig, ax = plt.subplots(ncols=3, figsize=(12, 4))
for iPid, pidType in enumerate(pidTypes):
Mrez3D = _bitdict_to_3Dmat(dataDB, dfJointDict, pidType, mice)
nChannel = Mrez3D.shape[0]
Mrez2D = np.sum(Mrez3D, axis=avgAxis) / (
nChannel - 2) # Target can't be either of the sources
print(np.max(Mrez2D))
imshow(fig,
ax[iPid],
Mrez2D,
title=pidType,
haveColorBar=True,
cmap='jet')
fig.suptitle('_'.join(key))
plt.show()
def correlation_by_session(dataDB,
datatype,
intervNames=None,
trialTypes=None,
minTrials=50):
for mousename in sorted(dataDB.mice):
for row, kwargs in _sweep_iter(dataDB,
mousename,
intervNames=intervNames,
trialTypes=trialTypes):
dataRSP = dataDB.get_neuro_data({'session': row['session']},
datatype=datatype,
**kwargs)[0]
dataRP = np.mean(dataRSP, axis=1)
nTrials, nChannel = dataRP.shape
if nTrials < minTrials:
print('Too few trials =', nTrials, ' for', row.values,
': skipping')
else:
corr = corr_2D(dataRP.T)
fig, ax = plt.subplots()
imshow(fig,
ax,
corr,
cmap='jet',
haveColorBar=True,
limits=[-1, 1])
plt.savefig('pics/corr_' + '_'.join(list(row.values)) + '.png')
plt.close()
def plot_area_clusters(self, fig, ax, regDict, haveLegend=False):
trgShape = self.areaSketch.shape + (3, )
colors = base_colors_rgb('tableau')
rez = np.zeros(trgShape)
imBinary = self.areaSketch == 1
imColor = np.outer(imBinary.astype(float),
np.array([0.1, 0.1, 0.1])).reshape(trgShape)
rez += imColor
# NOTE: Since number of clusters frequently exceeds number of discernable colors,
# The compromise is to drop all clusters of size 1 from the plot
regDictEff = {k: v for k, v in regDict.items() if len(v) > 1}
if len(regDictEff) > len(colors):
print('Warning: too many clusters to display, skipping',
len(regDictEff))
return
for iGroup, (label, lst) in enumerate(regDictEff.items()):
for iROI in lst:
imBinary = self.areaSketch == (iROI + 2)
imColor = np.outer(imBinary.astype(float),
colors[iGroup]).reshape(trgShape)
rez += imColor
rez = rgb_change_color(rez, [0, 0, 0], np.array([255, 255, 255]))
imshow(fig, ax, rez)
if haveLegend:
plt_add_fake_legend(ax, colors[:len(regDictEff)],
list(regDictEff.keys()))
def plot_2D_bytarget_synergy_cluster(dataDB,
h5fname,
dfSummary,
trgChName,
dropChannels=None,
clusterParam=1.0,
dropWeakChannelThr=None):
labels = dataDB.get_channel_labels()
trgIdx = labels.index(trgChName)
groupLst = sorted(list(set(dfSummary.columns) - {'key', 'mousename'}))
for key, dataMouse in dfSummary.groupby(groupLst):
mice = list(sorted(set(dataMouse['mousename'])))
dfJointDict = read_parse_joint_dataframe(dataMouse,
h5fname,
mice, ['syn'],
dropChannels=dropChannels)
nMice = len(mice)
fig, ax = plt.subplots(nrows=2, ncols=nMice, figsize=(4 * nMice, 8))
for iMouse, (idx, row) in enumerate(dataMouse.iterrows()):
Mrez3D = _bitdict_to_3Dmat(dataDB, dfJointDict, 'syn',
[row['mousename']])
Mrez2D = Mrez3D[:, :, trgIdx]
# Drop rows that are
if dropWeakChannelThr is not None:
keepIdxs = np.max(Mrez2D, axis=0) > dropWeakChannelThr
Mrez2DEff = Mrez2D[keepIdxs][:, keepIdxs]
else:
Mrez2DEff = Mrez2D
imshow(fig,
ax[0, iMouse],
Mrez2D,
title=row['mousename'],
haveColorBar=True,
cmap='jet')
clusters = cluster_dist_matrix_min(Mrez2DEff,
clusterParam,
method='OPTICS')
print(clusters)
cluster_plot(fig,
ax[1, iMouse],
Mrez2DEff,
clusters,
limits=None,
cmap='jet')
ax[0, 0].set_ylabel('Original')
ax[1, 0].set_ylabel('Clustered')
fig.suptitle('_'.join(key))
plt.show()
def imshow_dataset_by_mouse(dataDB,
ds,
dsetName,
plotNameSuffix='',
limits=None,
cmap='jet',
aspect=None,
fig1size=(5, 5),
havePerf=True,
dropRow=None,
dropCol=None):
resultDF = ds.list_dsets_pd()
fig, ax = plt.subplots(ncols=len(dataDB.mice),
figsize=(fig1size[0] * len(dataDB.mice),
fig1size[1]),
squeeze=False)
fig.suptitle(dsetName)
for iMouse, mousename in enumerate(sorted(dataDB.mice)):
queryDict = {"mousename": mousename, "name": dsetName}
data, attrs = ds.get_data_recent_by_query(queryDict, listDF=resultDF)
shapeLabels = attrs['target_dim'][1:-1].split(',')
mat = data.T
print(mat.shape)
if dropRow:
mat = np.delete(mat, dropRow, axis=0)
if dropCol:
mat = np.delete(mat, dropCol, axis=0)
# Plot data
imshow(fig,
ax[0][iMouse],
mat,
xlabel=shapeLabels[0],
ylabel=shapeLabels[1],
title=mousename,
haveColorBar=True,
limits=limits,
cmap=cmap,
aspect=aspect)
if havePerf:
thrIdx = dataDB.get_first_expert_session_idx(mousename)
if thrIdx is not None:
ax[0][iMouse].axvline(x=thrIdx, color='w', linestyle='--')
plt.savefig(dsetName + plotNameSuffix + '.pdf')
plt.show()
def imshow_dataset_by_session(dataDB,
ds,
dsetName,
plotNameSuffix='',
limits=None,
cmap='jet',
aspect=None,
colBased=True,
fig1size=(5, 5)):
resultDF = ds.list_dsets_pd()
for iMouse, mousename in enumerate(sorted(dataDB.mice)):
# Get data
queryDict = {"mousename": mousename, "name": dsetName}
data, attrs = ds.get_data_recent_by_query(queryDict, listDF=resultDF)
shapeLabels = attrs['target_dim']
assert data.ndim == 3, "Dataset must include sessions and 2 other dimensions"
# assert len(shapeLabels), "Dataset must include sessions and 2 other dimensions"
# assert shapeLabels[0] == 'sessions', "First dataset dimension must be sessions"
sessions = dataDB.get_sessions(mousename)
nSession = len(sessions)
if colBased:
fig, ax = plt.subplots(ncols=nSession,
figsize=(fig1size[0] * nSession,
fig1size[1]))
else:
fig, ax = plt.subplots(nrows=nSession,
figsize=(fig1size[0],
fig1size[1] * nSession))
# fig.suptitle(mousename)
for iSession, session in enumerate(sessions):
imshow(fig,
ax[iSession],
data[iSession],
xlabel=None,
ylabel=None,
title=session,
haveColorBar=True,
limits=limits,
cmap=cmap,
aspect=aspect)
plt.savefig(dsetName + "_" + mousename + plotNameSuffix + '.pdf')
plt.show()
def plot_area_clusters(self, fig, ax, regDict, haveLegend=False):
trgShape = self.allenMap.shape + (3,)
colors = base_colors_rgb('tableau')
rez = np.zeros(trgShape)
imBinary = self.allenMap == 0
imColor = np.outer(imBinary.astype(float), np.array([0.5, 0.5, 0.5])).reshape(trgShape)
rez += imColor
for iGroup, (label, lst) in enumerate(regDict.items()):
for iROI in lst:
imBinary = self.allenMap == self.allenIndices[iROI]
imColor = np.outer(imBinary.astype(float), colors[iGroup]).reshape(trgShape)
rez += imColor
imshow(fig, ax, rez)
if haveLegend:
plt_add_fake_legend(ax, colors[:len(regDict)], list(regDict.keys()))
def cluster_plot(fig,
ax,
M,
clusters,
channelLabels=None,
limits=None,
cmap='jet'):
'''
:param fig: Matplotlib figure
:param ax: Matplotlib axis
:param M: Distance matrix
:param clusters: Cluster labels
:param channelLabels: Channel labels
:param limits: Limits for the distance matrix plot
:param cmap: Color map
:return:
Plot distance matrix heatmap sorted by cluster. Plot separating lines for clusters
'''
idxs = np.argsort(clusters)
MSort = M[idxs][:, idxs]
idCluster, nCluster = np.unique(clusters, return_counts=True)
nClustCum = np.cumsum(nCluster)
imshow(fig,
ax,
MSort,
'clustering',
limits=limits,
haveColorBar=True,
cmap=cmap)
for nLine in nClustCum:
ax.axvline(x=nLine - 0.5, linestyle='--', color='black', alpha=0.3)
ax.axhline(y=nLine - 0.5, linestyle='--', color='black', alpha=0.3)
if channelLabels is not None:
cv = cluster_values(M, clusters)
for iClust in sorted(set(clusters)):
labelsThis = channelLabels[clusters == iClust]
print(iClust, ':', cv['avgCorr'][iClust], cv['avgRelCorr'][iClust],
':', labelsThis)
def plot_area_values(self, fig, ax, valLst, vmin=None, vmax=None, cmap='jet', haveColorBar=True):
# Mapping values to colors
vmin = vmin if vmin is not None else np.nanmin(valLst) * 0.9
vmax = vmax if vmax is not None else np.nanmax(valLst) * 1.1
colors = sample_cmap(cmap, valLst, vmin, vmax, dropAlpha=True)
trgShape = self.allenMap.shape + (3,)
rez = np.zeros(trgShape)
imBinary = self.allenMap == 0
imColor = np.outer(imBinary.astype(float), np.array([0.5, 0.5, 0.5])).reshape(trgShape)
rez += imColor
for iROI, color in enumerate(colors):
if not np.any(np.isnan(color)):
imBinary = self.allenMap == self.allenIndices[iROI]
imColor = np.outer(imBinary.astype(float), color).reshape(trgShape)
rez += imColor
rez = rgb_change_color(rez, [0, 0, 0], np.array([255, 255, 255]))
imshow(fig, ax, rez, haveColorBar=haveColorBar, limits=(vmin,vmax), cmap=cmap)
def plot_consistency_significant_activity_byaction(dataDB,
ds,
minTrials=10,
performance=None,
dropChannels=None,
metric='accuracy',
limits=None):
testFunc = test_metric_by_name(metric)
rows = ds.list_dsets_pd()
rows['mousename'] = [
dataDB.find_mouse_by_session(session) for session in rows['session']
]
dfColumns = ['datatype', 'phase', 'consistency']
dfConsistency = pd.DataFrame(columns=dfColumns)
for (datatype,
intervName), rowsMouse in rows.groupby(['datatype', 'intervName']):
pSigDict = {}
for mousename, rowsSession in rowsMouse.groupby(['mousename']):
pSig = []
for session, rowsTrial in rowsSession.groupby(['session']):
if (performance is None) or dataDB.is_matching_performance(
session, performance, mousename=mousename):
if len(rowsTrial) != 2:
print(mousename, session, rowsTrial)
raise ValueError('Expected exactly 2 rows')
dsetLabels = list(rowsTrial['dset'])
data1 = ds.get_data(dsetLabels[0])
data2 = ds.get_data(dsetLabels[1])
nTrials1 = data1.shape[0]
nTrials2 = data2.shape[1]
if (nTrials1 < minTrials) or (nTrials2 < minTrials):
print(session, datatype, intervName, 'too few trials',
nTrials1, nTrials2, ';; skipping')
else:
nChannels = data1.shape[1]
if dropChannels is not None:
channelMask = np.ones(nChannels).astype(bool)
channelMask[dropChannels] = 0
data1 = data1[:, channelMask]
data2 = data2[:, channelMask]
nChannels = nChannels - len(dropChannels)
pvals = [
testFunc(data1[:, iCh], data2[:, iCh])
for iCh in range(nChannels)
]
# pSig += [(np.array(pvals) < 0.01).astype(int)]
pSig += [-np.log10(np.array(pvals))]
# pSigDict[mousename] = np.sum(pSig, axis=0)
pSigDict[mousename] = np.mean(pSig, axis=0)
mice = sorted(pSigDict.keys())
nMice = len(mice)
corrCoef = np.zeros((nMice, nMice))
for iMouse, iName in enumerate(mice):
for jMouse, jName in enumerate(mice):
corrCoef[iMouse, jMouse] = np.corrcoef(pSigDict[iName],
pSigDict[jName])[0, 1]
plotSuffix = '_'.join([datatype, str(performance), intervName])
sns.pairplot(data=pd.DataFrame(pSigDict), vars=mice)
prefixPath = 'pics/consistency/significant_activity/byaction/bymouse/'
make_path(prefixPath)
plt.savefig(prefixPath + plotSuffix + '.svg')
plt.close()
fig2, ax2 = plt.subplots()
ax2.imshow(corrCoef, vmin=0, vmax=1)
imshow(fig2,
ax2,
corrCoef,
title='Significance Correlation',
haveColorBar=True,
limits=[0, 1],
xTicks=mice,
yTicks=mice)
prefixPath = 'pics/consistency/significant_activity/byaction/bymouse_corr/'
make_path(prefixPath)
plt.savefig(prefixPath + plotSuffix + '.svg')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(corrCoef)), 2)
dfConsistency = pd_append_row(dfConsistency,
[datatype, intervName, avgConsistency])
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistency, *dfColumns)
sns.heatmap(data=dfPivot, ax=ax, annot=True, vmax=1, cmap='jet')
prefixPath = 'pics/consistency/significant_activity/byaction/'
make_path(prefixPath)
fig.savefig(prefixPath + 'consistency_' + str(performance) + '.svg')
plt.close()
def plot_corr_s(dataDB, fig, ax, data, **plotKWArgs): # limits=[-1, 1]
if 'cmap' not in plotKWArgs.keys():
plotKWArgs['cmap'] = 'jet'
imshow(fig, ax, data, **plotKWArgs)
def plot_corr_mouse(dataDB,
mc,
estimator,
xParamName,
nDropPCA=None,
dropChannels=None,
haveBrain=False,
haveMono=True,
corrStrategy='mean',
exclQueryLst=None,
thrMono=0.4,
clusterParam=0.5,
fontsize=20,
**kwargs):
assert xParamName in ['intervName', 'trialType'], 'Unexpected parameter'
assert xParamName in kwargs.keys(), 'Requires ' + xParamName
dps = DataParameterSweep(dataDB, exclQueryLst, mousename='auto', **kwargs)
nMice = dps.param_size('mousename')
nXParam = dps.param_size(xParamName)
for paramVals, dfTmp in dps.sweepDF.groupby(
dps.invert_param(['mousename', xParamName])):
plotSuffix = param_vals_to_suffix(paramVals)
print(plotSuffix)
figCorr, axCorr = plt.subplots(nrows=nMice,
ncols=nXParam,
figsize=(4 * nXParam, 4 * nMice),
tight_layout=True)
figClust, axClust = plt.subplots(nrows=nMice,
ncols=nXParam,
figsize=(4 * nXParam, 4 * nMice),
tight_layout=True)
if haveBrain:
figBrain, axBrain = plt.subplots(nrows=nMice,
ncols=nXParam,
figsize=(4 * nXParam, 4 * nMice),
tight_layout=True)
if haveMono:
figMono, axMono = plt.subplots(nrows=nMice,
ncols=nXParam,
figsize=(4 * nXParam, 4 * nMice),
tight_layout=True)
for mousename, dfMouse in dfTmp.groupby(['mousename']):
iMouse = dps.param_index('mousename', mousename)
axCorr[iMouse][0].set_ylabel(mousename, fontsize=fontsize)
axClust[iMouse][0].set_ylabel(mousename, fontsize=fontsize)
if haveBrain:
axBrain[iMouse][0].set_ylabel(mousename, fontsize=fontsize)
if haveMono:
axMono[iMouse][0].set_ylabel(mousename, fontsize=fontsize)
for idx, row in dfMouse.iterrows():
xParamVal = row[xParamName]
iXParam = dps.param_index(xParamName, xParamVal)
axCorr[0][iXParam].set_title(xParamVal, fontsize=fontsize)
axClust[0][iXParam].set_title(xParamVal, fontsize=fontsize)
if haveBrain:
axBrain[0][iXParam].set_title(xParamVal, fontsize=fontsize)
if haveMono:
axMono[0][iXParam].set_title(xParamVal, fontsize=fontsize)
kwargsThis = pd_row_to_kwargs(row,
parseNone=True,
dropKeys=['mousename'])
channelLabels, rez2D = calc_corr_mouse(
dataDB,
mc,
mousename,
strategy=corrStrategy,
nDropPCA=nDropPCA,
dropChannels=dropChannels,
estimator=estimator,
**kwargsThis)
haveColorBar = iXParam == nXParam - 1
# Plot correlations
imshow(figCorr,
axCorr[iMouse][iXParam],
rez2D,
limits=[-1, 1],
cmap='jet',
haveColorBar=haveColorBar)
# Plot clustering
clusters = cluster_dist_matrix_max(rez2D,
clusterParam,
method='Affinity')
cluster_plot(figClust,
axClust[iMouse][iXParam],
rez2D,
clusters,
channelLabels,
limits=[-1, 1],
cmap='jet',
haveColorBar=haveColorBar)
if haveBrain:
cluster_brain_plot(figBrain,
axBrain[iMouse][iXParam],
dataDB,
clusters,
dropChannels=dropChannels)
if haveMono:
_plot_corr_1D(figMono, axMono[iMouse][iXParam],
channelLabels, rez2D, thrMono)
# Save image
prefixPrefixPath = 'pics/corr/mouse' + xParamName + '/dropPCA_' + str(
nDropPCA) + '/'
prefixPath = prefixPrefixPath + 'corr/'
make_path(prefixPath)
figCorr.savefig(prefixPath + 'corr_' + plotSuffix + '.svg')
plt.close(figCorr)
prefixPath = prefixPrefixPath + 'clust/'
make_path(prefixPath)
figClust.savefig(prefixPath + 'clust_' + plotSuffix + '.svg')
plt.close(figClust)
if haveBrain:
prefixPath = prefixPrefixPath + 'clust_brainplot/'
make_path(prefixPath)
figBrain.savefig(prefixPath + 'clust_brainplot_' + plotSuffix +
'.svg')
plt.close(figBrain)
if haveMono:
prefixPath = prefixPrefixPath + '1D/'
make_path(prefixPath)
figMono.savefig(prefixPath + '1Dplot_' + plotSuffix + '.svg')
plt.close(figMono)
def plot_corr_mousephase_submouse(dataDB,
mc,
estimator,
nDropPCA=None,
dropChannels=None,
exclQueryLst=None,
corrStrategy='mean',
fontsize=20,
**kwargs):
assert 'intervName' in kwargs.keys(), 'Requires phases'
dps = DataParameterSweep(dataDB, exclQueryLst, mousename='auto', **kwargs)
nMice = dps.param_size('mousename')
nInterv = dps.param_size('intervName')
for paramVals, dfTmp in dps.sweepDF.groupby(
dps.invert_param(['mousename', 'intervName'])):
plotSuffix = param_vals_to_suffix(paramVals)
print(plotSuffix)
figCorr, axCorr = plt.subplots(nrows=nMice,
ncols=nInterv,
figsize=(4 * nInterv, 4 * nMice))
for intervName, dfInterv in dfTmp.groupby(['intervName']):
iInterv = dps.param_index('intervName', intervName)
axCorr[0][iInterv].set_title(intervName, fontsize=fontsize)
rezDict = {}
for idx, row in dfInterv.iterrows():
mousename = row['mousename']
kwargsThis = pd_row_to_kwargs(row,
parseNone=True,
dropKeys=['mousename'])
channelLabels, rez2D = calc_corr_mouse(
dataDB,
mc,
mousename,
strategy=corrStrategy,
nDropPCA=nDropPCA,
dropChannels=dropChannels,
estimator=estimator,
**kwargsThis)
rezDict[mousename] = rez2D
# Plot correlations
rezMean = np.mean(list(rezDict.values()), axis=0)
for mousename, rezMouse in rezDict.items():
iMouse = dps.param_index('mousename', mousename)
axCorr[iMouse][0].set_ylabel(mousename, fontsize=fontsize)
if mousename in rezDict.keys():
haveColorBar = iInterv == nInterv - 1
imshow(figCorr,
axCorr[iMouse][iInterv],
rezMouse - rezMean,
title='corr',
haveColorBar=haveColorBar,
limits=[-1, 1],
cmap='RdBu_r')
# Save image
prefixPath = 'pics/corr/mousephase/dropPCA_' + str(
nDropPCA) + '/submouse/'
make_path(prefixPath)
figCorr.savefig(prefixPath + 'corr_' + plotSuffix + '.svg')
plt.close()
def plot_unique_top_pairs(dataDB,
h5fname,
dfSummary,
nTop=20,
dropChannels=None):
lmap = dataDB.map_channel_labels_canon()
dataLabels = dataDB.get_channel_labels()
dataLabelsCanon = [lmap[l] for l in dataLabels]
groupLst = sorted(list(set(dfSummary.columns) - {'key', 'mousename'}))
for key, dataMouse in dfSummary.groupby(groupLst):
mice = list(sorted(set(dataMouse['mousename'])))
dfJointDict = read_parse_joint_dataframe(dataMouse,
h5fname,
mice, ['unique'],
dropChannels=dropChannels)
fig, ax = plt.subplots(nrows=2,
ncols=len(mice),
figsize=(6 * len(mice), 8))
for iMouse, mousename in enumerate(mice):
Mrez3D = _bitdict_to_3Dmat(dataDB, dfJointDict, 'unique',
[mousename])
nChannel = Mrez3D.shape[0]
Mrez2D = np.sum(Mrez3D, axis=1) / (
nChannel - 2) # Target can't be either of the sources
imshow(fig,
ax[0][iMouse],
Mrez2D,
title=mousename,
ylabel='Unique2D',
haveColorBar=True,
cmap='jet')
# Find 2D indices of nTop strongest links
vals1D = Mrez2D.flatten()
idxsMax1D = np.argsort(vals1D)[::-1][:2 * nTop]
idxsMax2D = np.vstack(
[idxsMax1D // nChannel, idxsMax1D % nChannel]).T
valsMax1D = vals1D[idxsMax1D]
labelsMax2D = [(dataLabelsCanon[i], dataLabelsCanon[j])
for i, j in idxsMax2D]
valsMax1DUnpaired = []
labelsMax2DUnpaired = []
for l, v in zip(labelsMax2D, valsMax1D):
if str((l[1], l[0])) not in labelsMax2DUnpaired:
labelsMax2DUnpaired += [str(l)]
valsMax1DUnpaired += [v]
valsMax1DUnpaired = np.array(valsMax1DUnpaired)
labelsMax2DUnpaired = np.array(labelsMax2DUnpaired)
barplot_labeled(ax[1][iMouse],
valsMax1DUnpaired,
labelsMax2DUnpaired,
rotation=90)
idxsHigh = valsMax1DUnpaired > 0.5
print(mousename, labelsMax2DUnpaired[idxsHigh])
fig.suptitle('_'.join(key))
plt.show()
def pid_movie_2D_mousephase_bytrg(dataDB,
h5fname,
dfSummary,
trialTypes,
pidType,
trgChName,
vmin=None,
vmax=None,
fontsize=20):
labelsCanon = list(dataDB.map_channel_labels_canon().values())
nChannel = len(labelsCanon)
trgIdx = labelsCanon.index(trgChName)
mice = sorted(dataDB.mice)
nMice = len(mice)
nTrialType = len(trialTypes)
sweepCols = list(
set(dfSummary.columns) - {'mousename', 'trialType', 'key'})
for paramVals, dfTmp in dfSummary.groupby(sweepCols):
plotSuffix = param_vals_to_suffix(paramVals)
# Store all preprocessed data first
dataDict = {}
for mousename, dfMouse in dfTmp.groupby(['mousename']):
for idx, row in dfMouse.iterrows():
trialType = row['trialType']
print('Reading data, ', plotSuffix, mousename, trialType)
with h5py.File(h5fname, 'r') as f:
labels, data = pid_read_data(h5fname, row['key'])
# # Drop negatives
# vals = np.clip(vals, 0, None)
labels, data = get_pid_type(labels, data, pidType)
idxs = labels[:, 2] == trgIdx
dataDict[(mousename,
trialType)] = get_2D_map(labels[idxs, :2],
data[idxs], nChannel)
# Test that all datasets have the same duration
shapeSet = set([v.shape for v in dataDict.values()])
assert len(shapeSet) == 1
nTimes = shapeSet.pop()[0]
progBar = IntProgress(min=0, max=nTimes, description=plotSuffix)
display(progBar) # display the bar
for iTime in range(nTimes):
outfname = 'pid2D_byTrg_mousetrialtype_' + trgChName + '_' + plotSuffix + '_' + str(
iTime) + '.png'
if os.path.isfile(outfname):
print('Already calculated', iTime, 'skipping')
progBar.value += 1
continue
fig, ax = plt.subplots(nrows=nMice,
ncols=nTrialType,
figsize=(4 * nTrialType, 4 * nMice),
tight_layout=True)
for iMouse, mousename in enumerate(mice):
ax[iMouse][0].set_ylabel(mousename, fontsize=fontsize)
for iTT, trialType in enumerate(trialTypes):
ax[0][iTT].set_title(trialType, fontsize=fontsize)
# print(datatype, mousename)
dataPP = dataDict[(mousename, trialType)][iTime]
haveColorBar = iTT == nTrialType - 1
imshow(fig,
ax[iMouse][iTT],
dataPP,
limits=[vmin, vmax],
cmap='jet',
haveColorBar=haveColorBar)
plt.savefig(outfname)
plt.cla()
plt.clf()
plt.close('all')
progBar.value += 1
def plot_consistency_significant_activity_byphase(dataDB,
ds,
intervals,
minTrials=10,
performance=None,
dropChannels=None):
rows = ds.list_dsets_pd()
rows['mousename'] = [
dataDB.find_mouse_by_session(session) for session in rows['session']
]
dfColumns = ['datatype', 'trialType', 'consistency']
dfConsistency = pd.DataFrame(columns=dfColumns)
for (datatype,
trialType), rowsMouse in rows.groupby(['datatype', 'trialType']):
pSigDict = {}
for mousename, rowsSession in rowsMouse.groupby(['mousename']):
pSig = []
for session, rowsTrial in rowsSession.groupby(['session']):
if (performance is None) or dataDB.is_matching_performance(
session, performance, mousename=mousename):
assert intervals[0] in list(rowsTrial['intervName'])
assert intervals[1] in list(rowsTrial['intervName'])
dsetLabel1 = pd_is_one_row(
pd_query(rowsTrial,
{'intervName': intervals[0]}))[1]['dset']
dsetLabel2 = pd_is_one_row(
pd_query(rowsTrial,
{'intervName': intervals[1]}))[1]['dset']
data1 = ds.get_data(dsetLabel1)
data2 = ds.get_data(dsetLabel2)
nTrials1 = data1.shape[0]
nTrials2 = data2.shape[1]
if (nTrials1 < minTrials) or (nTrials2 < minTrials):
print(session, datatype, trialType, 'too few trials',
nTrials1, nTrials2, ';; skipping')
else:
nChannels = data1.shape[1]
if dropChannels is not None:
channelMask = np.ones(nChannels).astype(bool)
channelMask[dropChannels] = 0
data1 = data1[:, channelMask]
data2 = data2[:, channelMask]
nChannels = nChannels - len(dropChannels)
pvals = [
wilcoxon(data1[:, iCh],
data2[:, iCh],
alternative='two-sided')[1]
for iCh in range(nChannels)
]
# pSig += [(np.array(pvals) < 0.01).astype(int)]
pSig += [-np.log10(np.array(pvals))]
# pSigDict[mousename] = np.sum(pSig, axis=0)
pSigDict[mousename] = np.mean(pSig, axis=0)
mice = sorted(dataDB.mice)
nMice = len(mice)
corrCoef = np.zeros((nMice, nMice))
for iMouse, iName in enumerate(mice):
for jMouse, jName in enumerate(mice):
corrCoef[iMouse, jMouse] = np.corrcoef(pSigDict[iName],
pSigDict[jName])[0, 1]
sns.pairplot(data=pd.DataFrame(pSigDict), vars=mice)
prefixPath = 'pics/consistency/significant_activity/byphase/bymouse/'
make_path(prefixPath)
plt.savefig(prefixPath + datatype + '_' + trialType + '.svg')
plt.close()
fig2, ax2 = plt.subplots()
ax2.imshow(corrCoef, vmin=0, vmax=1)
imshow(fig2,
ax2,
corrCoef,
title='Significance Correlation',
haveColorBar=True,
limits=[0, 1],
xTicks=mice,
yTicks=mice)
prefixPath = 'pics/consistency/significant_activity/byphase/bymouse_corr/'
make_path(prefixPath)
plt.savefig(prefixPath + datatype + '_' + trialType + '.svg')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(corrCoef)), 2)
dfConsistency = pd_append_row(dfConsistency,
[datatype, trialType, avgConsistency])
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistency, *dfColumns)
sns.heatmap(data=dfPivot, ax=ax, annot=True, vmax=1, cmap='jet')
prefixPath = 'pics/consistency/significant_activity/byphase/'
make_path(prefixPath)
fig.savefig(prefixPath + 'consistency_' + str(performance) + '.svg')
plt.close()
def plot_pca_consistency(dataDB,
intervNames=None,
dropFirst=None,
dropChannels=None):
mice = sorted(dataDB.mice)
nMice = len(mice)
dfColumns = ['datatype', 'phase', 'consistency']
dfConsistency = pd.DataFrame(columns=dfColumns)
if intervNames is None:
intervNames = dataDB.get_interval_names()
for datatype in dataDB.get_data_types():
for intervName in intervNames:
fnameSuffix = datatype + '_' + intervName
print(fnameSuffix)
dataLst = []
for iMouse, mousename in enumerate(mice):
dataRSPLst = dataDB.get_neuro_data({'mousename': mousename},
datatype=datatype,
intervName=intervName)
print(set([d.shape[1:] for d in dataRSPLst]))
dataRSP = np.concatenate(dataRSPLst, axis=0)
dataRP = np.mean(dataRSP, axis=1)
dataRP = zscore(dataRP, axis=0)
if dropChannels is not None:
channelMask = np.ones(dataRP.shape[1]).astype(bool)
channelMask[dropChannels] = 0
dataRP = dataRP[:, channelMask]
dataLst += [dataRP]
fig, ax = plt.subplots(nrows=nMice,
ncols=nMice,
figsize=(4 * nMice, 4 * nMice))
rezMat = np.zeros((nMice, nMice))
for iMouse in range(nMice):
ax[iMouse][0].set_ylabel(mice[iMouse])
ax[-1][iMouse].set_xlabel(mice[iMouse])
for jMouse in range(nMice):
rezXY, rezYX, arrXX, arrXY, arrYY, arrYX = pca.paired_comparison(
dataLst[iMouse], dataLst[jMouse], dropFirst=dropFirst)
rezMat[iMouse][jMouse] = rezXY # np.mean([rezXY, rezYX])
ax[iMouse][jMouse].plot(arrXX, label='XX')
ax[iMouse][jMouse].plot(arrXY, label='XY')
ax[iMouse][jMouse].legend()
plt.savefig('pca_consistency_bymouse_evals_' + fnameSuffix +
'.svg')
plt.close()
fig, ax = plt.subplots()
imshow(fig,
ax,
rezMat,
haveColorBar=True,
limits=[0, 1],
xTicks=mice,
yTicks=mice)
plt.savefig('pca_consistency_bymouse_metric_' + fnameSuffix +
'.svg')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2)
dfConsistency = pd_append_row(
dfConsistency, [datatype, intervName, avgConsistency])
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistency, *dfColumns)
sns.heatmap(data=dfPivot, ax=ax, annot=True, vmin=0, vmax=1, cmap='jet')
fig.savefig('consistency_coactivity_metric.svg')
plt.close()
def plot_pca_alignment_bymouse(dataDB,
datatype='bn_session',
trialType=None,
intervName=None):
nMouse = len(dataDB.mice)
mice = sorted(dataDB.mice)
fig1, ax1 = plt.subplots(ncols=2, figsize=(8, 4))
fig2, ax2 = plt.subplots(nrows=nMouse,
ncols=nMouse,
figsize=(4 * nMouse, 4 * nMouse))
compLst = []
varLst = []
for mousename in sorted(dataDB.mice):
dataRSP = dataDB.get_neuro_data({'mousename': mousename},
intervName=intervName,
datatype=datatype,
trialType=trialType)
dataRSP = np.concatenate(dataRSP, axis=0)
dataRP = np.concatenate(dataRSP, axis=0) # Use timesteps as samples
pca = PCA(n_components=dataRP.shape[1])
pca.fit(dataRP)
ax1[0].semilogy(pca.explained_variance_ratio_, label=mousename)
compLst += [np.copy(pca.components_)]
varLst += [np.sqrt(pca.explained_variance_ratio_)]
# Compute pca_alignment coefficient
matAlign = np.zeros((nMouse, nMouse))
for iMouse in range(nMouse):
ax2[iMouse][0].set_ylabel(mice[iMouse])
ax2[-1][iMouse].set_xlabel(mice[iMouse])
for jMouse in range(nMouse):
matVar = np.outer(varLst[iMouse], varLst[jMouse])
matComp = np.abs(compLst[iMouse].dot(compLst[jMouse].T))
matTot = matVar * matComp**2
matAlign[iMouse, jMouse] = np.sum(matTot)
print(np.sum(matComp**2))
ax2[iMouse, jMouse].imshow(matComp, vmin=0, vmax=1)
ax1[0].set_xlabel('PCA')
ax1[0].set_ylabel('Explained Variance')
imshow(fig1,
ax1[1],
matAlign,
limits=[0, 1],
title='PCA alignment',
xTicks=mice,
yTicks=mice,
haveColorBar=True,
cmap='jet')
ax1[0].legend()
fig1.savefig('PCA_alignment_values_' + datatype + '_' + str(trialType) +
'.svg')
fig2.savefig('PCA_alignment_matrix_' + datatype + '_' + str(trialType) +
'.svg')
plt.close()
def plot_corr_consistency_l1_phase(dataDB,
nDropPCA=None,
dropChannels=None,
performance=None,
datatype=None):
mice = sorted(dataDB.mice)
phases = dataDB.get_interval_names()
nPhases = len(phases)
dfColumns = ['mousename', 'trialtype', 'consistency']
dfConsistency = pd.DataFrame(columns=dfColumns)
for iMouse, mousename in enumerate(mice):
for trialType in dataDB.get_trial_type_names():
fnameSuffix = '_'.join(
[datatype, mousename, trialType,
str(performance)])
print(fnameSuffix)
corrLst = []
for intervName in phases:
kwargs = {
'datatype': datatype,
'intervName': intervName,
'trialType': trialType
}
if performance is not None:
kwargs['performance'] = performance
dataRSPLst = dataDB.get_neuro_data({'mousename': mousename},
zscoreDim='rs',
**kwargs)
dataRSP = np.concatenate(dataRSPLst, axis=0)
dataRP = np.mean(dataRSP, axis=1)
# dataRP = zscore(dataRP, axis=0)
if dropChannels is not None:
channelMask = np.ones(dataRP.shape[1]).astype(bool)
channelMask[dropChannels] = 0
dataRP = dataRP[:, channelMask]
if nDropPCA is not None:
dataRP = drop_PCA(dataRP, nDropPCA)
corrLst += [tril_1D(np.corrcoef(dataRP.T))]
# fig, ax = plt.subplots(nrows=nMice, ncols=nMice, figsize=(4 * nMice, 4 * nMice))
pairDict = {}
rezMat = np.zeros((nPhases, nPhases))
for idxNamei, iName in enumerate(phases):
pairDict[iName] = corrLst[idxNamei]
for idxNamej, jName in enumerate(phases):
rezMat[idxNamei][idxNamej] = np.corrcoef(
corrLst[idxNamei], corrLst[idxNamej])[0, 1]
pPlot = sns.pairplot(data=pd.DataFrame(pairDict),
vars=phases,
kind='kde')
prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str(
nDropPCA) + '/scatter/'
make_path(prefixPath)
plt.savefig(prefixPath + 'scatter_' + fnameSuffix + '.svg')
plt.close()
fig, ax = plt.subplots()
imshow(fig,
ax,
rezMat,
haveColorBar=True,
limits=[0, 1],
xTicks=phases,
yTicks=phases,
cmap='jet')
prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str(
nDropPCA) + '/metric/'
make_path(prefixPath)
plt.savefig(prefixPath + 'metric_' + fnameSuffix + '.svg')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2)
dfConsistency = pd_append_row(
dfConsistency, [mousename, trialType, avgConsistency])
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistency, *dfColumns)
sns.heatmap(data=dfPivot, ax=ax, annot=True, vmin=0, vmax=1, cmap='jet')
prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str(nDropPCA) + '/'
make_path(prefixPath)
fig.savefig(prefixPath + datatype + '_' + str(performance) + '.svg')
plt.close()
def plot_corr_consistency_l1_mouse(
dataDB,
nDropPCA=None,
dropChannels=None,
exclQueryLst=None,
**kwargs): # performances=None, trialTypes=None,
assert 'intervName' in kwargs.keys(), 'Requires phases'
dps = DataParameterSweep(dataDB,
exclQueryLst,
mousename='auto',
intervName='auto',
datatype='auto',
**kwargs)
mice = sorted(dataDB.mice)
nMice = len(mice)
for paramExtraVals, dfTmp in dps.sweepDF.groupby(
dps.invert_param(['mousename', 'datatype', 'intervName'])):
plotExtraSuffix = param_vals_to_suffix(paramExtraVals)
dfColumns = ['datatype', 'phase', 'consistency']
dfConsistency = pd.DataFrame(columns=dfColumns)
for paramVals, dfMouse in dfTmp.groupby(dps.invert_param(['mousename'
])):
plotSuffix = param_vals_to_suffix(paramVals)
print(plotSuffix)
corrLst = []
for idx, row in dfMouse.iterrows():
plotSuffix = '_'.join([str(s) for s in row.values])
print(plotSuffix)
kwargsThis = pd_row_to_kwargs(row,
parseNone=True,
dropKeys=['mousename'])
dataRSPLst = dataDB.get_neuro_data(
{
'mousename': row['mousename']
}, # NOTE: zscore channels for each session to avoid session-wise effects
zscoreDim='rs',
**kwargsThis)
dataRSP = np.concatenate(dataRSPLst, axis=0)
dataRP = np.mean(dataRSP, axis=1)
# dataRP = zscore(dataRP, axis=0)
if dropChannels is not None:
channelMask = np.ones(dataRP.shape[1]).astype(bool)
channelMask[dropChannels] = 0
dataRP = dataRP[:, channelMask]
if nDropPCA is not None:
dataRP = drop_PCA(dataRP, nDropPCA)
corrLst += [tril_1D(np.corrcoef(dataRP.T))]
# fig, ax = plt.subplots(nrows=nMice, ncols=nMice, figsize=(4 * nMice, 4 * nMice))
pairDict = {}
rezMat = np.zeros((nMice, nMice))
for iMouse, mousename in enumerate(mice):
# ax[iMouse][0].set_ylabel(mice[iMouse])
# ax[-1][iMouse].set_xlabel(mice[iMouse])
pairDict[mousename] = corrLst[iMouse]
for jMouse in range(nMice):
# rezMat[iMouse][jMouse] = 1 - rmae(corrLst[iMouse], corrLst[jMouse])
rezMat[iMouse][jMouse] = np.corrcoef(
corrLst[iMouse], corrLst[jMouse])[0, 1]
# cci = offdiag_1D(corrLst[iMouse])
# ccj = offdiag_1D(corrLst[jMouse])
# ax[iMouse][jMouse].plot(cci, ccj, '.')
pPlot = sns.pairplot(data=pd.DataFrame(pairDict),
vars=mice,
kind='kde')
prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str(
nDropPCA) + '/scatter/'
make_path(prefixPath)
plt.savefig(prefixPath + 'scatter_' + plotSuffix + '.svg')
plt.close()
fig, ax = plt.subplots()
imshow(fig,
ax,
rezMat,
haveColorBar=True,
limits=[0, 1],
xTicks=mice,
yTicks=mice,
cmap='jet')
prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str(
nDropPCA) + '/metric/'
make_path(prefixPath)
plt.savefig(prefixPath + 'metric_' + plotSuffix + '.svg')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2)
dfConsistency = pd_append_row(
dfConsistency,
[row['datatype'], row['intervName'], avgConsistency])
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistency, *dfColumns)
sns.heatmap(data=dfPivot,
ax=ax,
annot=True,
vmin=0,
vmax=1,
cmap='jet')
prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str(
nDropPCA) + '/'
make_path(prefixPath)
fig.savefig(prefixPath + plotExtraSuffix + '.svg')
plt.close()
def plot_corr_mouse_2DF(dfDict,
mc,
estimator,
intervNameMap,
intervOrdMap,
corrStrategy='mean',
nDropPCA=None,
dropChannels=None,
exclQueryLst=None):
dataDBTmp = list(dfDict.values())[0]
mice = sorted(dataDBTmp.mice)
nMice = len(mice)
intervNames = dataDBTmp.get_interval_names()
trialTypes = dataDBTmp.get_trial_type_names()
for trialType in trialTypes:
for intervName in intervNames:
intervLabel = intervName if intervName not in intervNameMap else intervNameMap[
intervName]
plotSuffix = trialType + '_' + intervLabel
print(plotSuffix)
fig, ax = plt.subplots(nrows=2,
ncols=nMice,
figsize=(4 * nMice, 4 * 2),
tight_layout=True)
for iDB, (dbName, dataDB) in enumerate(dfDict.items()):
ax[iDB][0].set_ylabel(dbName)
intervEffName = intervName if (
dbName, intervName) not in intervOrdMap else intervOrdMap[(
dbName, intervName)]
for iMouse, mousename in enumerate(mice):
ax[0][iMouse].set_title(mousename)
kwargs = {
'mousename': mousename,
'intervName': intervEffName,
'trialType': trialType,
'datatype': 'bn_session'
}
if np.all([
not subset_dict(excl, kwargs)
for excl in exclQueryLst
]):
del kwargs['mousename']
kwargs = {
k: v if v != 'None' else None
for k, v in kwargs.items()
}
channelLabels, rez2D = calc_corr_mouse(
dataDB,
mc,
mousename,
strategy=corrStrategy,
nDropPCA=nDropPCA,
dropChannels=dropChannels,
estimator=estimator,
**kwargs)
imshow(fig,
ax[iDB][iMouse],
rez2D,
limits=[-1, 1],
cmap='jet',
haveColorBar=iMouse == nMice - 1)
# Save image
prefixPath = 'pics/corr/bystim/dropPCA_' + str(nDropPCA) + '/'
make_path(prefixPath)
plt.savefig(prefixPath + 'corr_bn_session_' + plotSuffix + '.svg')
plt.close()
