def plot_2D_bytarget(dataDB, h5fname, dfSummary, trgChName, dropChannels=None): pidTypes = ['unique', 'syn', 'red'] labels = dataDB.get_channel_labels() trgIdx = labels.index(trgChName) groupLst = sorted(list(set(dfSummary.columns) - {'key', 'mousename'})) for key, dataMouse in dfSummary.groupby(groupLst): mice = list(sorted(set(dataMouse['mousename']))) dfJointDict = read_parse_joint_dataframe(dataMouse, h5fname, mice, pidTypes, dropChannels=dropChannels) fig, ax = plt.subplots(ncols=3, figsize=(12, 4)) for iPid, pidType in enumerate(pidTypes): Mrez3D = _bitdict_to_3Dmat(dataDB, dfJointDict, pidType, mice) nChannel = Mrez3D.shape[0] Mrez2D = Mrez3D[:, :, trgIdx] print(np.max(Mrez2D)) imshow(fig, ax[iPid], Mrez2D, title=pidType, haveColorBar=True, cmap='jet') fig.suptitle('_'.join(key)) plt.show()
def plot_2D_avg(dataDB, h5fname, dfSummary, dropChannels=None, avgAxis=2): pidTypes = ['unique', 'syn', 'red'] groupLst = sorted(list(set(dfSummary.columns) - {'key', 'mousename'})) for key, dataMouse in dfSummary.groupby(groupLst): mice = list(sorted(set(dataMouse['mousename']))) dfJointDict = read_parse_joint_dataframe(dataMouse, h5fname, mice, pidTypes, dropChannels=dropChannels) fig, ax = plt.subplots(ncols=3, figsize=(12, 4)) for iPid, pidType in enumerate(pidTypes): Mrez3D = _bitdict_to_3Dmat(dataDB, dfJointDict, pidType, mice) nChannel = Mrez3D.shape[0] Mrez2D = np.sum(Mrez3D, axis=avgAxis) / ( nChannel - 2) # Target can't be either of the sources print(np.max(Mrez2D)) imshow(fig, ax[iPid], Mrez2D, title=pidType, haveColorBar=True, cmap='jet') fig.suptitle('_'.join(key)) plt.show()
def correlation_by_session(dataDB, datatype, intervNames=None, trialTypes=None, minTrials=50): for mousename in sorted(dataDB.mice): for row, kwargs in _sweep_iter(dataDB, mousename, intervNames=intervNames, trialTypes=trialTypes): dataRSP = dataDB.get_neuro_data({'session': row['session']}, datatype=datatype, **kwargs)[0] dataRP = np.mean(dataRSP, axis=1) nTrials, nChannel = dataRP.shape if nTrials < minTrials: print('Too few trials =', nTrials, ' for', row.values, ': skipping') else: corr = corr_2D(dataRP.T) fig, ax = plt.subplots() imshow(fig, ax, corr, cmap='jet', haveColorBar=True, limits=[-1, 1]) plt.savefig('pics/corr_' + '_'.join(list(row.values)) + '.png') plt.close()
def plot_area_clusters(self, fig, ax, regDict, haveLegend=False): trgShape = self.areaSketch.shape + (3, ) colors = base_colors_rgb('tableau') rez = np.zeros(trgShape) imBinary = self.areaSketch == 1 imColor = np.outer(imBinary.astype(float), np.array([0.1, 0.1, 0.1])).reshape(trgShape) rez += imColor # NOTE: Since number of clusters frequently exceeds number of discernable colors, # The compromise is to drop all clusters of size 1 from the plot regDictEff = {k: v for k, v in regDict.items() if len(v) > 1} if len(regDictEff) > len(colors): print('Warning: too many clusters to display, skipping', len(regDictEff)) return for iGroup, (label, lst) in enumerate(regDictEff.items()): for iROI in lst: imBinary = self.areaSketch == (iROI + 2) imColor = np.outer(imBinary.astype(float), colors[iGroup]).reshape(trgShape) rez += imColor rez = rgb_change_color(rez, [0, 0, 0], np.array([255, 255, 255])) imshow(fig, ax, rez) if haveLegend: plt_add_fake_legend(ax, colors[:len(regDictEff)], list(regDictEff.keys()))
def plot_2D_bytarget_synergy_cluster(dataDB, h5fname, dfSummary, trgChName, dropChannels=None, clusterParam=1.0, dropWeakChannelThr=None): labels = dataDB.get_channel_labels() trgIdx = labels.index(trgChName) groupLst = sorted(list(set(dfSummary.columns) - {'key', 'mousename'})) for key, dataMouse in dfSummary.groupby(groupLst): mice = list(sorted(set(dataMouse['mousename']))) dfJointDict = read_parse_joint_dataframe(dataMouse, h5fname, mice, ['syn'], dropChannels=dropChannels) nMice = len(mice) fig, ax = plt.subplots(nrows=2, ncols=nMice, figsize=(4 * nMice, 8)) for iMouse, (idx, row) in enumerate(dataMouse.iterrows()): Mrez3D = _bitdict_to_3Dmat(dataDB, dfJointDict, 'syn', [row['mousename']]) Mrez2D = Mrez3D[:, :, trgIdx] # Drop rows that are if dropWeakChannelThr is not None: keepIdxs = np.max(Mrez2D, axis=0) > dropWeakChannelThr Mrez2DEff = Mrez2D[keepIdxs][:, keepIdxs] else: Mrez2DEff = Mrez2D imshow(fig, ax[0, iMouse], Mrez2D, title=row['mousename'], haveColorBar=True, cmap='jet') clusters = cluster_dist_matrix_min(Mrez2DEff, clusterParam, method='OPTICS') print(clusters) cluster_plot(fig, ax[1, iMouse], Mrez2DEff, clusters, limits=None, cmap='jet') ax[0, 0].set_ylabel('Original') ax[1, 0].set_ylabel('Clustered') fig.suptitle('_'.join(key)) plt.show()
def imshow_dataset_by_mouse(dataDB, ds, dsetName, plotNameSuffix='', limits=None, cmap='jet', aspect=None, fig1size=(5, 5), havePerf=True, dropRow=None, dropCol=None): resultDF = ds.list_dsets_pd() fig, ax = plt.subplots(ncols=len(dataDB.mice), figsize=(fig1size[0] * len(dataDB.mice), fig1size[1]), squeeze=False) fig.suptitle(dsetName) for iMouse, mousename in enumerate(sorted(dataDB.mice)): queryDict = {"mousename": mousename, "name": dsetName} data, attrs = ds.get_data_recent_by_query(queryDict, listDF=resultDF) shapeLabels = attrs['target_dim'][1:-1].split(',') mat = data.T print(mat.shape) if dropRow: mat = np.delete(mat, dropRow, axis=0) if dropCol: mat = np.delete(mat, dropCol, axis=0) # Plot data imshow(fig, ax[0][iMouse], mat, xlabel=shapeLabels[0], ylabel=shapeLabels[1], title=mousename, haveColorBar=True, limits=limits, cmap=cmap, aspect=aspect) if havePerf: thrIdx = dataDB.get_first_expert_session_idx(mousename) if thrIdx is not None: ax[0][iMouse].axvline(x=thrIdx, color='w', linestyle='--') plt.savefig(dsetName + plotNameSuffix + '.pdf') plt.show()
def imshow_dataset_by_session(dataDB, ds, dsetName, plotNameSuffix='', limits=None, cmap='jet', aspect=None, colBased=True, fig1size=(5, 5)): resultDF = ds.list_dsets_pd() for iMouse, mousename in enumerate(sorted(dataDB.mice)): # Get data queryDict = {"mousename": mousename, "name": dsetName} data, attrs = ds.get_data_recent_by_query(queryDict, listDF=resultDF) shapeLabels = attrs['target_dim'] assert data.ndim == 3, "Dataset must include sessions and 2 other dimensions" # assert len(shapeLabels), "Dataset must include sessions and 2 other dimensions" # assert shapeLabels[0] == 'sessions', "First dataset dimension must be sessions" sessions = dataDB.get_sessions(mousename) nSession = len(sessions) if colBased: fig, ax = plt.subplots(ncols=nSession, figsize=(fig1size[0] * nSession, fig1size[1])) else: fig, ax = plt.subplots(nrows=nSession, figsize=(fig1size[0], fig1size[1] * nSession)) # fig.suptitle(mousename) for iSession, session in enumerate(sessions): imshow(fig, ax[iSession], data[iSession], xlabel=None, ylabel=None, title=session, haveColorBar=True, limits=limits, cmap=cmap, aspect=aspect) plt.savefig(dsetName + "_" + mousename + plotNameSuffix + '.pdf') plt.show()
def plot_area_clusters(self, fig, ax, regDict, haveLegend=False): trgShape = self.allenMap.shape + (3,) colors = base_colors_rgb('tableau') rez = np.zeros(trgShape) imBinary = self.allenMap == 0 imColor = np.outer(imBinary.astype(float), np.array([0.5, 0.5, 0.5])).reshape(trgShape) rez += imColor for iGroup, (label, lst) in enumerate(regDict.items()): for iROI in lst: imBinary = self.allenMap == self.allenIndices[iROI] imColor = np.outer(imBinary.astype(float), colors[iGroup]).reshape(trgShape) rez += imColor imshow(fig, ax, rez) if haveLegend: plt_add_fake_legend(ax, colors[:len(regDict)], list(regDict.keys()))
def cluster_plot(fig, ax, M, clusters, channelLabels=None, limits=None, cmap='jet'): ''' :param fig: Matplotlib figure :param ax: Matplotlib axis :param M: Distance matrix :param clusters: Cluster labels :param channelLabels: Channel labels :param limits: Limits for the distance matrix plot :param cmap: Color map :return: Plot distance matrix heatmap sorted by cluster. Plot separating lines for clusters ''' idxs = np.argsort(clusters) MSort = M[idxs][:, idxs] idCluster, nCluster = np.unique(clusters, return_counts=True) nClustCum = np.cumsum(nCluster) imshow(fig, ax, MSort, 'clustering', limits=limits, haveColorBar=True, cmap=cmap) for nLine in nClustCum: ax.axvline(x=nLine - 0.5, linestyle='--', color='black', alpha=0.3) ax.axhline(y=nLine - 0.5, linestyle='--', color='black', alpha=0.3) if channelLabels is not None: cv = cluster_values(M, clusters) for iClust in sorted(set(clusters)): labelsThis = channelLabels[clusters == iClust] print(iClust, ':', cv['avgCorr'][iClust], cv['avgRelCorr'][iClust], ':', labelsThis)
def plot_area_values(self, fig, ax, valLst, vmin=None, vmax=None, cmap='jet', haveColorBar=True): # Mapping values to colors vmin = vmin if vmin is not None else np.nanmin(valLst) * 0.9 vmax = vmax if vmax is not None else np.nanmax(valLst) * 1.1 colors = sample_cmap(cmap, valLst, vmin, vmax, dropAlpha=True) trgShape = self.allenMap.shape + (3,) rez = np.zeros(trgShape) imBinary = self.allenMap == 0 imColor = np.outer(imBinary.astype(float), np.array([0.5, 0.5, 0.5])).reshape(trgShape) rez += imColor for iROI, color in enumerate(colors): if not np.any(np.isnan(color)): imBinary = self.allenMap == self.allenIndices[iROI] imColor = np.outer(imBinary.astype(float), color).reshape(trgShape) rez += imColor rez = rgb_change_color(rez, [0, 0, 0], np.array([255, 255, 255])) imshow(fig, ax, rez, haveColorBar=haveColorBar, limits=(vmin,vmax), cmap=cmap)
def plot_consistency_significant_activity_byaction(dataDB, ds, minTrials=10, performance=None, dropChannels=None, metric='accuracy', limits=None): testFunc = test_metric_by_name(metric) rows = ds.list_dsets_pd() rows['mousename'] = [ dataDB.find_mouse_by_session(session) for session in rows['session'] ] dfColumns = ['datatype', 'phase', 'consistency'] dfConsistency = pd.DataFrame(columns=dfColumns) for (datatype, intervName), rowsMouse in rows.groupby(['datatype', 'intervName']): pSigDict = {} for mousename, rowsSession in rowsMouse.groupby(['mousename']): pSig = [] for session, rowsTrial in rowsSession.groupby(['session']): if (performance is None) or dataDB.is_matching_performance( session, performance, mousename=mousename): if len(rowsTrial) != 2: print(mousename, session, rowsTrial) raise ValueError('Expected exactly 2 rows') dsetLabels = list(rowsTrial['dset']) data1 = ds.get_data(dsetLabels[0]) data2 = ds.get_data(dsetLabels[1]) nTrials1 = data1.shape[0] nTrials2 = data2.shape[1] if (nTrials1 < minTrials) or (nTrials2 < minTrials): print(session, datatype, intervName, 'too few trials', nTrials1, nTrials2, ';; skipping') else: nChannels = data1.shape[1] if dropChannels is not None: channelMask = np.ones(nChannels).astype(bool) channelMask[dropChannels] = 0 data1 = data1[:, channelMask] data2 = data2[:, channelMask] nChannels = nChannels - len(dropChannels) pvals = [ testFunc(data1[:, iCh], data2[:, iCh]) for iCh in range(nChannels) ] # pSig += [(np.array(pvals) < 0.01).astype(int)] pSig += [-np.log10(np.array(pvals))] # pSigDict[mousename] = np.sum(pSig, axis=0) pSigDict[mousename] = np.mean(pSig, axis=0) mice = sorted(pSigDict.keys()) nMice = len(mice) corrCoef = np.zeros((nMice, nMice)) for iMouse, iName in enumerate(mice): for jMouse, jName in enumerate(mice): corrCoef[iMouse, jMouse] = np.corrcoef(pSigDict[iName], pSigDict[jName])[0, 1] plotSuffix = '_'.join([datatype, str(performance), intervName]) sns.pairplot(data=pd.DataFrame(pSigDict), vars=mice) prefixPath = 'pics/consistency/significant_activity/byaction/bymouse/' make_path(prefixPath) plt.savefig(prefixPath + plotSuffix + '.svg') plt.close() fig2, ax2 = plt.subplots() ax2.imshow(corrCoef, vmin=0, vmax=1) imshow(fig2, ax2, corrCoef, title='Significance Correlation', haveColorBar=True, limits=[0, 1], xTicks=mice, yTicks=mice) prefixPath = 'pics/consistency/significant_activity/byaction/bymouse_corr/' make_path(prefixPath) plt.savefig(prefixPath + plotSuffix + '.svg') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(corrCoef)), 2) dfConsistency = pd_append_row(dfConsistency, [datatype, intervName, avgConsistency]) fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistency, *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, vmax=1, cmap='jet') prefixPath = 'pics/consistency/significant_activity/byaction/' make_path(prefixPath) fig.savefig(prefixPath + 'consistency_' + str(performance) + '.svg') plt.close()
def plot_corr_s(dataDB, fig, ax, data, **plotKWArgs): # limits=[-1, 1] if 'cmap' not in plotKWArgs.keys(): plotKWArgs['cmap'] = 'jet' imshow(fig, ax, data, **plotKWArgs)
def plot_corr_mouse(dataDB, mc, estimator, xParamName, nDropPCA=None, dropChannels=None, haveBrain=False, haveMono=True, corrStrategy='mean', exclQueryLst=None, thrMono=0.4, clusterParam=0.5, fontsize=20, **kwargs): assert xParamName in ['intervName', 'trialType'], 'Unexpected parameter' assert xParamName in kwargs.keys(), 'Requires ' + xParamName dps = DataParameterSweep(dataDB, exclQueryLst, mousename='auto', **kwargs) nMice = dps.param_size('mousename') nXParam = dps.param_size(xParamName) for paramVals, dfTmp in dps.sweepDF.groupby( dps.invert_param(['mousename', xParamName])): plotSuffix = param_vals_to_suffix(paramVals) print(plotSuffix) figCorr, axCorr = plt.subplots(nrows=nMice, ncols=nXParam, figsize=(4 * nXParam, 4 * nMice), tight_layout=True) figClust, axClust = plt.subplots(nrows=nMice, ncols=nXParam, figsize=(4 * nXParam, 4 * nMice), tight_layout=True) if haveBrain: figBrain, axBrain = plt.subplots(nrows=nMice, ncols=nXParam, figsize=(4 * nXParam, 4 * nMice), tight_layout=True) if haveMono: figMono, axMono = plt.subplots(nrows=nMice, ncols=nXParam, figsize=(4 * nXParam, 4 * nMice), tight_layout=True) for mousename, dfMouse in dfTmp.groupby(['mousename']): iMouse = dps.param_index('mousename', mousename) axCorr[iMouse][0].set_ylabel(mousename, fontsize=fontsize) axClust[iMouse][0].set_ylabel(mousename, fontsize=fontsize) if haveBrain: axBrain[iMouse][0].set_ylabel(mousename, fontsize=fontsize) if haveMono: axMono[iMouse][0].set_ylabel(mousename, fontsize=fontsize) for idx, row in dfMouse.iterrows(): xParamVal = row[xParamName] iXParam = dps.param_index(xParamName, xParamVal) axCorr[0][iXParam].set_title(xParamVal, fontsize=fontsize) axClust[0][iXParam].set_title(xParamVal, fontsize=fontsize) if haveBrain: axBrain[0][iXParam].set_title(xParamVal, fontsize=fontsize) if haveMono: axMono[0][iXParam].set_title(xParamVal, fontsize=fontsize) kwargsThis = pd_row_to_kwargs(row, parseNone=True, dropKeys=['mousename']) channelLabels, rez2D = calc_corr_mouse( dataDB, mc, mousename, strategy=corrStrategy, nDropPCA=nDropPCA, dropChannels=dropChannels, estimator=estimator, **kwargsThis) haveColorBar = iXParam == nXParam - 1 # Plot correlations imshow(figCorr, axCorr[iMouse][iXParam], rez2D, limits=[-1, 1], cmap='jet', haveColorBar=haveColorBar) # Plot clustering clusters = cluster_dist_matrix_max(rez2D, clusterParam, method='Affinity') cluster_plot(figClust, axClust[iMouse][iXParam], rez2D, clusters, channelLabels, limits=[-1, 1], cmap='jet', haveColorBar=haveColorBar) if haveBrain: cluster_brain_plot(figBrain, axBrain[iMouse][iXParam], dataDB, clusters, dropChannels=dropChannels) if haveMono: _plot_corr_1D(figMono, axMono[iMouse][iXParam], channelLabels, rez2D, thrMono) # Save image prefixPrefixPath = 'pics/corr/mouse' + xParamName + '/dropPCA_' + str( nDropPCA) + '/' prefixPath = prefixPrefixPath + 'corr/' make_path(prefixPath) figCorr.savefig(prefixPath + 'corr_' + plotSuffix + '.svg') plt.close(figCorr) prefixPath = prefixPrefixPath + 'clust/' make_path(prefixPath) figClust.savefig(prefixPath + 'clust_' + plotSuffix + '.svg') plt.close(figClust) if haveBrain: prefixPath = prefixPrefixPath + 'clust_brainplot/' make_path(prefixPath) figBrain.savefig(prefixPath + 'clust_brainplot_' + plotSuffix + '.svg') plt.close(figBrain) if haveMono: prefixPath = prefixPrefixPath + '1D/' make_path(prefixPath) figMono.savefig(prefixPath + '1Dplot_' + plotSuffix + '.svg') plt.close(figMono)
def plot_corr_mousephase_submouse(dataDB, mc, estimator, nDropPCA=None, dropChannels=None, exclQueryLst=None, corrStrategy='mean', fontsize=20, **kwargs): assert 'intervName' in kwargs.keys(), 'Requires phases' dps = DataParameterSweep(dataDB, exclQueryLst, mousename='auto', **kwargs) nMice = dps.param_size('mousename') nInterv = dps.param_size('intervName') for paramVals, dfTmp in dps.sweepDF.groupby( dps.invert_param(['mousename', 'intervName'])): plotSuffix = param_vals_to_suffix(paramVals) print(plotSuffix) figCorr, axCorr = plt.subplots(nrows=nMice, ncols=nInterv, figsize=(4 * nInterv, 4 * nMice)) for intervName, dfInterv in dfTmp.groupby(['intervName']): iInterv = dps.param_index('intervName', intervName) axCorr[0][iInterv].set_title(intervName, fontsize=fontsize) rezDict = {} for idx, row in dfInterv.iterrows(): mousename = row['mousename'] kwargsThis = pd_row_to_kwargs(row, parseNone=True, dropKeys=['mousename']) channelLabels, rez2D = calc_corr_mouse( dataDB, mc, mousename, strategy=corrStrategy, nDropPCA=nDropPCA, dropChannels=dropChannels, estimator=estimator, **kwargsThis) rezDict[mousename] = rez2D # Plot correlations rezMean = np.mean(list(rezDict.values()), axis=0) for mousename, rezMouse in rezDict.items(): iMouse = dps.param_index('mousename', mousename) axCorr[iMouse][0].set_ylabel(mousename, fontsize=fontsize) if mousename in rezDict.keys(): haveColorBar = iInterv == nInterv - 1 imshow(figCorr, axCorr[iMouse][iInterv], rezMouse - rezMean, title='corr', haveColorBar=haveColorBar, limits=[-1, 1], cmap='RdBu_r') # Save image prefixPath = 'pics/corr/mousephase/dropPCA_' + str( nDropPCA) + '/submouse/' make_path(prefixPath) figCorr.savefig(prefixPath + 'corr_' + plotSuffix + '.svg') plt.close()
def plot_unique_top_pairs(dataDB, h5fname, dfSummary, nTop=20, dropChannels=None): lmap = dataDB.map_channel_labels_canon() dataLabels = dataDB.get_channel_labels() dataLabelsCanon = [lmap[l] for l in dataLabels] groupLst = sorted(list(set(dfSummary.columns) - {'key', 'mousename'})) for key, dataMouse in dfSummary.groupby(groupLst): mice = list(sorted(set(dataMouse['mousename']))) dfJointDict = read_parse_joint_dataframe(dataMouse, h5fname, mice, ['unique'], dropChannels=dropChannels) fig, ax = plt.subplots(nrows=2, ncols=len(mice), figsize=(6 * len(mice), 8)) for iMouse, mousename in enumerate(mice): Mrez3D = _bitdict_to_3Dmat(dataDB, dfJointDict, 'unique', [mousename]) nChannel = Mrez3D.shape[0] Mrez2D = np.sum(Mrez3D, axis=1) / ( nChannel - 2) # Target can't be either of the sources imshow(fig, ax[0][iMouse], Mrez2D, title=mousename, ylabel='Unique2D', haveColorBar=True, cmap='jet') # Find 2D indices of nTop strongest links vals1D = Mrez2D.flatten() idxsMax1D = np.argsort(vals1D)[::-1][:2 * nTop] idxsMax2D = np.vstack( [idxsMax1D // nChannel, idxsMax1D % nChannel]).T valsMax1D = vals1D[idxsMax1D] labelsMax2D = [(dataLabelsCanon[i], dataLabelsCanon[j]) for i, j in idxsMax2D] valsMax1DUnpaired = [] labelsMax2DUnpaired = [] for l, v in zip(labelsMax2D, valsMax1D): if str((l[1], l[0])) not in labelsMax2DUnpaired: labelsMax2DUnpaired += [str(l)] valsMax1DUnpaired += [v] valsMax1DUnpaired = np.array(valsMax1DUnpaired) labelsMax2DUnpaired = np.array(labelsMax2DUnpaired) barplot_labeled(ax[1][iMouse], valsMax1DUnpaired, labelsMax2DUnpaired, rotation=90) idxsHigh = valsMax1DUnpaired > 0.5 print(mousename, labelsMax2DUnpaired[idxsHigh]) fig.suptitle('_'.join(key)) plt.show()
def pid_movie_2D_mousephase_bytrg(dataDB, h5fname, dfSummary, trialTypes, pidType, trgChName, vmin=None, vmax=None, fontsize=20): labelsCanon = list(dataDB.map_channel_labels_canon().values()) nChannel = len(labelsCanon) trgIdx = labelsCanon.index(trgChName) mice = sorted(dataDB.mice) nMice = len(mice) nTrialType = len(trialTypes) sweepCols = list( set(dfSummary.columns) - {'mousename', 'trialType', 'key'}) for paramVals, dfTmp in dfSummary.groupby(sweepCols): plotSuffix = param_vals_to_suffix(paramVals) # Store all preprocessed data first dataDict = {} for mousename, dfMouse in dfTmp.groupby(['mousename']): for idx, row in dfMouse.iterrows(): trialType = row['trialType'] print('Reading data, ', plotSuffix, mousename, trialType) with h5py.File(h5fname, 'r') as f: labels, data = pid_read_data(h5fname, row['key']) # # Drop negatives # vals = np.clip(vals, 0, None) labels, data = get_pid_type(labels, data, pidType) idxs = labels[:, 2] == trgIdx dataDict[(mousename, trialType)] = get_2D_map(labels[idxs, :2], data[idxs], nChannel) # Test that all datasets have the same duration shapeSet = set([v.shape for v in dataDict.values()]) assert len(shapeSet) == 1 nTimes = shapeSet.pop()[0] progBar = IntProgress(min=0, max=nTimes, description=plotSuffix) display(progBar) # display the bar for iTime in range(nTimes): outfname = 'pid2D_byTrg_mousetrialtype_' + trgChName + '_' + plotSuffix + '_' + str( iTime) + '.png' if os.path.isfile(outfname): print('Already calculated', iTime, 'skipping') progBar.value += 1 continue fig, ax = plt.subplots(nrows=nMice, ncols=nTrialType, figsize=(4 * nTrialType, 4 * nMice), tight_layout=True) for iMouse, mousename in enumerate(mice): ax[iMouse][0].set_ylabel(mousename, fontsize=fontsize) for iTT, trialType in enumerate(trialTypes): ax[0][iTT].set_title(trialType, fontsize=fontsize) # print(datatype, mousename) dataPP = dataDict[(mousename, trialType)][iTime] haveColorBar = iTT == nTrialType - 1 imshow(fig, ax[iMouse][iTT], dataPP, limits=[vmin, vmax], cmap='jet', haveColorBar=haveColorBar) plt.savefig(outfname) plt.cla() plt.clf() plt.close('all') progBar.value += 1
def plot_consistency_significant_activity_byphase(dataDB, ds, intervals, minTrials=10, performance=None, dropChannels=None): rows = ds.list_dsets_pd() rows['mousename'] = [ dataDB.find_mouse_by_session(session) for session in rows['session'] ] dfColumns = ['datatype', 'trialType', 'consistency'] dfConsistency = pd.DataFrame(columns=dfColumns) for (datatype, trialType), rowsMouse in rows.groupby(['datatype', 'trialType']): pSigDict = {} for mousename, rowsSession in rowsMouse.groupby(['mousename']): pSig = [] for session, rowsTrial in rowsSession.groupby(['session']): if (performance is None) or dataDB.is_matching_performance( session, performance, mousename=mousename): assert intervals[0] in list(rowsTrial['intervName']) assert intervals[1] in list(rowsTrial['intervName']) dsetLabel1 = pd_is_one_row( pd_query(rowsTrial, {'intervName': intervals[0]}))[1]['dset'] dsetLabel2 = pd_is_one_row( pd_query(rowsTrial, {'intervName': intervals[1]}))[1]['dset'] data1 = ds.get_data(dsetLabel1) data2 = ds.get_data(dsetLabel2) nTrials1 = data1.shape[0] nTrials2 = data2.shape[1] if (nTrials1 < minTrials) or (nTrials2 < minTrials): print(session, datatype, trialType, 'too few trials', nTrials1, nTrials2, ';; skipping') else: nChannels = data1.shape[1] if dropChannels is not None: channelMask = np.ones(nChannels).astype(bool) channelMask[dropChannels] = 0 data1 = data1[:, channelMask] data2 = data2[:, channelMask] nChannels = nChannels - len(dropChannels) pvals = [ wilcoxon(data1[:, iCh], data2[:, iCh], alternative='two-sided')[1] for iCh in range(nChannels) ] # pSig += [(np.array(pvals) < 0.01).astype(int)] pSig += [-np.log10(np.array(pvals))] # pSigDict[mousename] = np.sum(pSig, axis=0) pSigDict[mousename] = np.mean(pSig, axis=0) mice = sorted(dataDB.mice) nMice = len(mice) corrCoef = np.zeros((nMice, nMice)) for iMouse, iName in enumerate(mice): for jMouse, jName in enumerate(mice): corrCoef[iMouse, jMouse] = np.corrcoef(pSigDict[iName], pSigDict[jName])[0, 1] sns.pairplot(data=pd.DataFrame(pSigDict), vars=mice) prefixPath = 'pics/consistency/significant_activity/byphase/bymouse/' make_path(prefixPath) plt.savefig(prefixPath + datatype + '_' + trialType + '.svg') plt.close() fig2, ax2 = plt.subplots() ax2.imshow(corrCoef, vmin=0, vmax=1) imshow(fig2, ax2, corrCoef, title='Significance Correlation', haveColorBar=True, limits=[0, 1], xTicks=mice, yTicks=mice) prefixPath = 'pics/consistency/significant_activity/byphase/bymouse_corr/' make_path(prefixPath) plt.savefig(prefixPath + datatype + '_' + trialType + '.svg') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(corrCoef)), 2) dfConsistency = pd_append_row(dfConsistency, [datatype, trialType, avgConsistency]) fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistency, *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, vmax=1, cmap='jet') prefixPath = 'pics/consistency/significant_activity/byphase/' make_path(prefixPath) fig.savefig(prefixPath + 'consistency_' + str(performance) + '.svg') plt.close()
def plot_pca_consistency(dataDB, intervNames=None, dropFirst=None, dropChannels=None): mice = sorted(dataDB.mice) nMice = len(mice) dfColumns = ['datatype', 'phase', 'consistency'] dfConsistency = pd.DataFrame(columns=dfColumns) if intervNames is None: intervNames = dataDB.get_interval_names() for datatype in dataDB.get_data_types(): for intervName in intervNames: fnameSuffix = datatype + '_' + intervName print(fnameSuffix) dataLst = [] for iMouse, mousename in enumerate(mice): dataRSPLst = dataDB.get_neuro_data({'mousename': mousename}, datatype=datatype, intervName=intervName) print(set([d.shape[1:] for d in dataRSPLst])) dataRSP = np.concatenate(dataRSPLst, axis=0) dataRP = np.mean(dataRSP, axis=1) dataRP = zscore(dataRP, axis=0) if dropChannels is not None: channelMask = np.ones(dataRP.shape[1]).astype(bool) channelMask[dropChannels] = 0 dataRP = dataRP[:, channelMask] dataLst += [dataRP] fig, ax = plt.subplots(nrows=nMice, ncols=nMice, figsize=(4 * nMice, 4 * nMice)) rezMat = np.zeros((nMice, nMice)) for iMouse in range(nMice): ax[iMouse][0].set_ylabel(mice[iMouse]) ax[-1][iMouse].set_xlabel(mice[iMouse]) for jMouse in range(nMice): rezXY, rezYX, arrXX, arrXY, arrYY, arrYX = pca.paired_comparison( dataLst[iMouse], dataLst[jMouse], dropFirst=dropFirst) rezMat[iMouse][jMouse] = rezXY # np.mean([rezXY, rezYX]) ax[iMouse][jMouse].plot(arrXX, label='XX') ax[iMouse][jMouse].plot(arrXY, label='XY') ax[iMouse][jMouse].legend() plt.savefig('pca_consistency_bymouse_evals_' + fnameSuffix + '.svg') plt.close() fig, ax = plt.subplots() imshow(fig, ax, rezMat, haveColorBar=True, limits=[0, 1], xTicks=mice, yTicks=mice) plt.savefig('pca_consistency_bymouse_metric_' + fnameSuffix + '.svg') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2) dfConsistency = pd_append_row( dfConsistency, [datatype, intervName, avgConsistency]) fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistency, *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, vmin=0, vmax=1, cmap='jet') fig.savefig('consistency_coactivity_metric.svg') plt.close()
def plot_pca_alignment_bymouse(dataDB, datatype='bn_session', trialType=None, intervName=None): nMouse = len(dataDB.mice) mice = sorted(dataDB.mice) fig1, ax1 = plt.subplots(ncols=2, figsize=(8, 4)) fig2, ax2 = plt.subplots(nrows=nMouse, ncols=nMouse, figsize=(4 * nMouse, 4 * nMouse)) compLst = [] varLst = [] for mousename in sorted(dataDB.mice): dataRSP = dataDB.get_neuro_data({'mousename': mousename}, intervName=intervName, datatype=datatype, trialType=trialType) dataRSP = np.concatenate(dataRSP, axis=0) dataRP = np.concatenate(dataRSP, axis=0) # Use timesteps as samples pca = PCA(n_components=dataRP.shape[1]) pca.fit(dataRP) ax1[0].semilogy(pca.explained_variance_ratio_, label=mousename) compLst += [np.copy(pca.components_)] varLst += [np.sqrt(pca.explained_variance_ratio_)] # Compute pca_alignment coefficient matAlign = np.zeros((nMouse, nMouse)) for iMouse in range(nMouse): ax2[iMouse][0].set_ylabel(mice[iMouse]) ax2[-1][iMouse].set_xlabel(mice[iMouse]) for jMouse in range(nMouse): matVar = np.outer(varLst[iMouse], varLst[jMouse]) matComp = np.abs(compLst[iMouse].dot(compLst[jMouse].T)) matTot = matVar * matComp**2 matAlign[iMouse, jMouse] = np.sum(matTot) print(np.sum(matComp**2)) ax2[iMouse, jMouse].imshow(matComp, vmin=0, vmax=1) ax1[0].set_xlabel('PCA') ax1[0].set_ylabel('Explained Variance') imshow(fig1, ax1[1], matAlign, limits=[0, 1], title='PCA alignment', xTicks=mice, yTicks=mice, haveColorBar=True, cmap='jet') ax1[0].legend() fig1.savefig('PCA_alignment_values_' + datatype + '_' + str(trialType) + '.svg') fig2.savefig('PCA_alignment_matrix_' + datatype + '_' + str(trialType) + '.svg') plt.close()
def plot_corr_consistency_l1_phase(dataDB, nDropPCA=None, dropChannels=None, performance=None, datatype=None): mice = sorted(dataDB.mice) phases = dataDB.get_interval_names() nPhases = len(phases) dfColumns = ['mousename', 'trialtype', 'consistency'] dfConsistency = pd.DataFrame(columns=dfColumns) for iMouse, mousename in enumerate(mice): for trialType in dataDB.get_trial_type_names(): fnameSuffix = '_'.join( [datatype, mousename, trialType, str(performance)]) print(fnameSuffix) corrLst = [] for intervName in phases: kwargs = { 'datatype': datatype, 'intervName': intervName, 'trialType': trialType } if performance is not None: kwargs['performance'] = performance dataRSPLst = dataDB.get_neuro_data({'mousename': mousename}, zscoreDim='rs', **kwargs) dataRSP = np.concatenate(dataRSPLst, axis=0) dataRP = np.mean(dataRSP, axis=1) # dataRP = zscore(dataRP, axis=0) if dropChannels is not None: channelMask = np.ones(dataRP.shape[1]).astype(bool) channelMask[dropChannels] = 0 dataRP = dataRP[:, channelMask] if nDropPCA is not None: dataRP = drop_PCA(dataRP, nDropPCA) corrLst += [tril_1D(np.corrcoef(dataRP.T))] # fig, ax = plt.subplots(nrows=nMice, ncols=nMice, figsize=(4 * nMice, 4 * nMice)) pairDict = {} rezMat = np.zeros((nPhases, nPhases)) for idxNamei, iName in enumerate(phases): pairDict[iName] = corrLst[idxNamei] for idxNamej, jName in enumerate(phases): rezMat[idxNamei][idxNamej] = np.corrcoef( corrLst[idxNamei], corrLst[idxNamej])[0, 1] pPlot = sns.pairplot(data=pd.DataFrame(pairDict), vars=phases, kind='kde') prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str( nDropPCA) + '/scatter/' make_path(prefixPath) plt.savefig(prefixPath + 'scatter_' + fnameSuffix + '.svg') plt.close() fig, ax = plt.subplots() imshow(fig, ax, rezMat, haveColorBar=True, limits=[0, 1], xTicks=phases, yTicks=phases, cmap='jet') prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str( nDropPCA) + '/metric/' make_path(prefixPath) plt.savefig(prefixPath + 'metric_' + fnameSuffix + '.svg') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2) dfConsistency = pd_append_row( dfConsistency, [mousename, trialType, avgConsistency]) fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistency, *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, vmin=0, vmax=1, cmap='jet') prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str(nDropPCA) + '/' make_path(prefixPath) fig.savefig(prefixPath + datatype + '_' + str(performance) + '.svg') plt.close()
def plot_corr_consistency_l1_mouse( dataDB, nDropPCA=None, dropChannels=None, exclQueryLst=None, **kwargs): # performances=None, trialTypes=None, assert 'intervName' in kwargs.keys(), 'Requires phases' dps = DataParameterSweep(dataDB, exclQueryLst, mousename='auto', intervName='auto', datatype='auto', **kwargs) mice = sorted(dataDB.mice) nMice = len(mice) for paramExtraVals, dfTmp in dps.sweepDF.groupby( dps.invert_param(['mousename', 'datatype', 'intervName'])): plotExtraSuffix = param_vals_to_suffix(paramExtraVals) dfColumns = ['datatype', 'phase', 'consistency'] dfConsistency = pd.DataFrame(columns=dfColumns) for paramVals, dfMouse in dfTmp.groupby(dps.invert_param(['mousename' ])): plotSuffix = param_vals_to_suffix(paramVals) print(plotSuffix) corrLst = [] for idx, row in dfMouse.iterrows(): plotSuffix = '_'.join([str(s) for s in row.values]) print(plotSuffix) kwargsThis = pd_row_to_kwargs(row, parseNone=True, dropKeys=['mousename']) dataRSPLst = dataDB.get_neuro_data( { 'mousename': row['mousename'] }, # NOTE: zscore channels for each session to avoid session-wise effects zscoreDim='rs', **kwargsThis) dataRSP = np.concatenate(dataRSPLst, axis=0) dataRP = np.mean(dataRSP, axis=1) # dataRP = zscore(dataRP, axis=0) if dropChannels is not None: channelMask = np.ones(dataRP.shape[1]).astype(bool) channelMask[dropChannels] = 0 dataRP = dataRP[:, channelMask] if nDropPCA is not None: dataRP = drop_PCA(dataRP, nDropPCA) corrLst += [tril_1D(np.corrcoef(dataRP.T))] # fig, ax = plt.subplots(nrows=nMice, ncols=nMice, figsize=(4 * nMice, 4 * nMice)) pairDict = {} rezMat = np.zeros((nMice, nMice)) for iMouse, mousename in enumerate(mice): # ax[iMouse][0].set_ylabel(mice[iMouse]) # ax[-1][iMouse].set_xlabel(mice[iMouse]) pairDict[mousename] = corrLst[iMouse] for jMouse in range(nMice): # rezMat[iMouse][jMouse] = 1 - rmae(corrLst[iMouse], corrLst[jMouse]) rezMat[iMouse][jMouse] = np.corrcoef( corrLst[iMouse], corrLst[jMouse])[0, 1] # cci = offdiag_1D(corrLst[iMouse]) # ccj = offdiag_1D(corrLst[jMouse]) # ax[iMouse][jMouse].plot(cci, ccj, '.') pPlot = sns.pairplot(data=pd.DataFrame(pairDict), vars=mice, kind='kde') prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str( nDropPCA) + '/scatter/' make_path(prefixPath) plt.savefig(prefixPath + 'scatter_' + plotSuffix + '.svg') plt.close() fig, ax = plt.subplots() imshow(fig, ax, rezMat, haveColorBar=True, limits=[0, 1], xTicks=mice, yTicks=mice, cmap='jet') prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str( nDropPCA) + '/metric/' make_path(prefixPath) plt.savefig(prefixPath + 'metric_' + plotSuffix + '.svg') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2) dfConsistency = pd_append_row( dfConsistency, [row['datatype'], row['intervName'], avgConsistency]) fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistency, *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, vmin=0, vmax=1, cmap='jet') prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str( nDropPCA) + '/' make_path(prefixPath) fig.savefig(prefixPath + plotExtraSuffix + '.svg') plt.close()
def plot_corr_mouse_2DF(dfDict, mc, estimator, intervNameMap, intervOrdMap, corrStrategy='mean', nDropPCA=None, dropChannels=None, exclQueryLst=None): dataDBTmp = list(dfDict.values())[0] mice = sorted(dataDBTmp.mice) nMice = len(mice) intervNames = dataDBTmp.get_interval_names() trialTypes = dataDBTmp.get_trial_type_names() for trialType in trialTypes: for intervName in intervNames: intervLabel = intervName if intervName not in intervNameMap else intervNameMap[ intervName] plotSuffix = trialType + '_' + intervLabel print(plotSuffix) fig, ax = plt.subplots(nrows=2, ncols=nMice, figsize=(4 * nMice, 4 * 2), tight_layout=True) for iDB, (dbName, dataDB) in enumerate(dfDict.items()): ax[iDB][0].set_ylabel(dbName) intervEffName = intervName if ( dbName, intervName) not in intervOrdMap else intervOrdMap[( dbName, intervName)] for iMouse, mousename in enumerate(mice): ax[0][iMouse].set_title(mousename) kwargs = { 'mousename': mousename, 'intervName': intervEffName, 'trialType': trialType, 'datatype': 'bn_session' } if np.all([ not subset_dict(excl, kwargs) for excl in exclQueryLst ]): del kwargs['mousename'] kwargs = { k: v if v != 'None' else None for k, v in kwargs.items() } channelLabels, rez2D = calc_corr_mouse( dataDB, mc, mousename, strategy=corrStrategy, nDropPCA=nDropPCA, dropChannels=dropChannels, estimator=estimator, **kwargs) imshow(fig, ax[iDB][iMouse], rez2D, limits=[-1, 1], cmap='jet', haveColorBar=iMouse == nMice - 1) # Save image prefixPath = 'pics/corr/bystim/dropPCA_' + str(nDropPCA) + '/' make_path(prefixPath) plt.savefig(prefixPath + 'corr_bn_session_' + plotSuffix + '.svg') plt.close()