def find_parse_data_paths(self, pathAUD): for mouseName, dfMouse in self.dfSession.groupby(['mousename']): pathMouse = os.path.join(pathAUD, mouseName) prepcommon._testdir(pathMouse) for dayName, dfDay in dfMouse.groupby(['dateKey']): print(mouseName, dayName) pathDay = os.path.join(pathMouse, dayName, 'widefield_labview') pathAllen = os.path.join(pathDay, 'ROI_Allen', 'registration_transform_1510.mat') haveDir = prepcommon._testdir(pathDay, critical=False) haveFile = prepcommon._testfile(pathAllen, critical=False) if haveDir and haveFile: self.pathT1[mouseName][dayName] = pathAllen for idx, row in dfDay.iterrows(): sessionSuffix = row['sessionKey'] pathSession = os.path.join(pathDay, sessionSuffix) pathMat = os.path.join(pathSession, 'Matt_files') # pathTrialInd = os.path.join(pathMat, 'trials_ind.mat') matFiles = [ f for f in os.listdir(pathMat) if os.path.splitext(f)[1] == '.mat' and f[:9] == 'Behavior_' ] assert len(matFiles) == 1 pathTrialStruct = os.path.join(pathMat, matFiles[0]) pathActivePassive = os.path.join( pathMat, 'trials_with_and_wo_initial_moves_OCIA_from_movie.mat' ) pathMovementVectors = os.path.join( pathMat, 'move_vectors_from_movie.mat') prepcommon._testdir(pathSession) prepcommon._testdir(pathMat) # prepcommon._testfile(pathTrialInd) prepcommon._testfile(pathTrialStruct) prepcommon._testfile(pathActivePassive) prepcommon._testfile(pathMovementVectors, critical=False) self.dataPaths = pd_append_row( self.dataPaths, [ mouseName, dayName, sessionSuffix, pathSession, #pathTrialInd, pathTrialStruct, pathActivePassive, pathMovementVectors ])
def process_metadata_files(self, pwd): for mouseName, mouseRows in self.dataPaths.groupby(['mouse']): h5name = os.path.join(pwd, mouseName + '.h5') prepcommon._h5_append_group(h5name, 'metadataTrial') prepcommon._h5_append_group(h5name, 'accuracy') prepcommon._h5_append_group(h5name, 'dprime') dfMetaSession = pd.DataFrame(columns=['session', 'timestamp', 'delay']) for idx, row in mouseRows.iterrows(): sessionName = row['day'] + '_' + row['session'] with h5py.File(h5name, 'a') as h5f: metaProcessed = sessionName in h5f['metadataTrial'].keys() print(mouseName, sessionName) if metaProcessed: print('-- already processed') continue # Read metadata from two file types and data, reconcile with h5py.File(h5name, 'r') as f: dataRSPShape = f['data'][sessionName].shape dfTrialStructMat = self.read_trial_structure_as_pd(row['trialStructPathMat'], mouseName) timeStamp, dfTrialStructLabview = self.read_parse_labview_trial_structure_as_pd(row['trialStructPathLabview']) delay, dfTrialStruct = self.reconcile_trial_structure(dfTrialStructMat, dfTrialStructLabview, dataRSPShape) # Store to file dfTrialStruct.to_hdf(h5name, '/metadataTrial/' + sessionName) dfMetaSession = pd_append_row(dfMetaSession, [sessionName, timeStamp, delay]) # Calculate and store accuracy and dprime ttDict = prepcommon.count_trial_types(dfTrialStruct) print(ttDict) acc = accuracy(ttDict['Hit'], ttDict['Miss'], ttDict['FA'], ttDict['CR']) dp = d_prime(ttDict['Hit'], ttDict['Miss'], ttDict['FA'], ttDict['CR']) with h5py.File(h5name, 'a') as h5f: h5f['accuracy'].create_dataset(sessionName, data=acc) h5f['dprime'].create_dataset(sessionName, data=dp) dfMetaSession.to_hdf(h5name, 'metadataSession')
def extract_timestamps_video(self, pwd): for mouseName, dfMouse in self.dataPaths.groupby(['mouse']): h5name = os.path.join(pwd, mouseName + '.h5') for idx, row in dfMouse.iterrows(): sessionName = row['day'] + '_' + row['session'] print(mouseName, sessionName) filePaths = self.parse_video_paths(row['sessionPath'], row['day']) fileBases = [os.path.basename(f) for f in filePaths] timeStampStrings = [ ':'.join( [f[:4], f[4:6], f[6:8], f[9:11], f[11:13], f[13:15]]) for f in fileBases ] timeStamps = [ datetime.strptime(f, "%Y:%d:%m:%H:%M:%S") for f in timeStampStrings ] metadata = pd.read_hdf(h5name, '/metadata/' + sessionName) if len(timeStamps) != len(metadata): print(mouseName, sessionName, len(timeStamps), len(metadata)) if len(timeStamps) < len(metadata): timeStamps += [None] * (len(metadata) - len(timeStamps)) elif len(timeStamps) == len(metadata) + 1: metadata = pd_append_row(metadata, [None] * len(metadata.columns)) print(metadata.tail()) metadata['timeStamps'] = timeStamps metadata.to_hdf(h5name, '/metadata/' + sessionName)
def plot_all_frac_significant_performance_scatter(dataDB, h5fname, minTrials=50): pidTypes = ['unique', 'red', 'syn'] summaryDF = pid_all_summary_df(h5fname) for keyLst, dfSession in summaryDF.groupby(['datatype', 'phase']): # 'mousename' keyLabel = '_'.join(keyLst) pidDFAll = pd.DataFrame(columns=['pid', 'x', 'y', 'mouse']) pidDictNaive = defaultdict(list) pidDictExpert = defaultdict(list) fig, ax = plt.subplots(nrows=2, ncols=len(pidTypes), figsize=(len(pidTypes) * 4, 8), tight_layout=True) for keyLst2, dfSession2 in dfSession.groupby(['mousename' ]): # 'mousename' print(keyLabel, keyLst2) for idx, row in dfSession2.iterrows(): nTrialsThis = dataDB.get_ntrial_bytype( {'session': dfSession['session']}, trialType='iGO') if nTrialsThis < minTrials: print('Skipping session', dfSession['session'], 'because it has too few trials', nTrialsThis) else: perf = dataDB.get_performance(row['session']) df1 = pd.read_hdf(h5fname, key=row['key']) # Merge Unique1 and Unique2 for this plot df1 = preprocess_unique(df1) for pidType in pidTypes: pVal = df1[df1['PID'] == pidType]['p'] fracSig = np.mean(pVal < 0.01) pidDFAll = pd_append_row( pidDFAll, [pidType, perf, fracSig, keyLst2]) # pidDFAll[pidType] += [(perf, fracSig, keyLst2)] if perf < 0.7: pidDictNaive[pidType] += [fracSig] else: pidDictExpert[pidType] += [fracSig] for iFig, pidType in enumerate(pidTypes): ax[0, iFig].set_title(pidType) ax[0, iFig].set_xlabel('Performance') ax[0, iFig].set_ylabel('Fraction Significant') dataThis = pidDFAll[pidDFAll['pid'] == pidType] sns.scatterplot(ax=ax[0, iFig], data=dataThis, x='x', y='y', hue='mouse') sns.regplot(ax=ax[0, iFig], data=dataThis, x='x', y='y', scatter=False) ax[0, iFig].set_ylim(-0.01, 1.01) # sns.regplot(x=x, y=y, ax=ax[0, iFig]) # ax[0, iFig].plot(*np.array(pidDictSession[pidType]).T, '.', label=str(keyLst2)) for iFig, pidType in enumerate(pidTypes): ax[0, iFig].legend() ax[0, iFig].set_xlim([0, 1]) ax[0, iFig].axvline(x=0.7, linestyle='--', color='gray', alpha=0.5) violins_labeled(ax[1, iFig], [pidDictNaive[pidType], pidDictExpert[pidType]], ['naive', 'expert'], 'performance', 'Fraction Significant', violinInner='box', style='bar', sigTestPairs=[[0, 1]]) ax[1, iFig].set_ylim([0, 1]) # plt.show() plt.savefig('PID_Freq_Significant_vs_perf_' + keyLabel + '.png') plt.close()
def plot_consistency_bytrialtype(h5fname, dfSummary, dropChannels=None, performance=None, datatype=None, trialTypes=None, kind='point', fisherThr=0.1, limits=None): pidTypes = ['unique', 'syn', 'red'] limitKWargs = { 'vmin': limits[0], 'vmax': limits[1] } if limits is not None else {} if performance is None: dfSummaryEff = pd_query(dfSummary, {'datatype': datatype}) else: dfSummaryEff = pd_query(dfSummary, { 'datatype': datatype, 'performance': performance }) dfColumns = ['mousename', 'phase', 'consistency'] dfConsistencyDict = { pidType: pd.DataFrame(columns=dfColumns) for pidType in pidTypes } for (mousename, phase), df1 in dfSummaryEff.groupby(['mousename', 'phase']): fnameSuffix = '_'.join([mousename, datatype, phase, str(performance)]) trialTypes = trialTypes if trialTypes is not None else sorted( list(set(df1['trialType']))) nTrialTypes = len(trialTypes) dfTrialTypeDict = {} for iPid, pidType in enumerate(pidTypes): dfTrialTypeDict[pidType] = pd.DataFrame() for trialType, dfTrialType in df1.groupby(['trialType']): if trialType in trialTypes: dfTmp = read_parse_joint_dataframe(dfTrialType, h5fname, [mousename], pidTypes, dropChannels=dropChannels) for iPid, pidType in enumerate(pidTypes): dfTrialTypeDict[pidType][trialType] = dfTmp[pidType][ 'muTrue_' + mousename] for iPid, pidType in enumerate(pidTypes): maxRange = 0.35 if pidType == 'syn' else 1.0 # As consistency metric perform Fischer's exact test for significant vs unsignificant links # As pairplot show contingency tables if kind == 'fisher': fischerLabels = ['low', 'high'] rezMat = np.full((nTrialTypes, nTrialTypes), np.nan) fig, ax = plt.subplots(nrows=nTrialTypes, ncols=nTrialTypes, figsize=(4 * nTrialTypes, 4 * nTrialTypes)) for idxTTi, iTT in enumerate(trialTypes): ax[idxTTi, 0].set_ylabel(iTT) ax[-1, idxTTi].set_xlabel(iTT) for idxTTj, jTT in enumerate(trialTypes): if idxTTi == idxTTj: ax[idxTTi][idxTTj].hist( dfTrialTypeDict[pidType][iTT], range=[0, maxRange], bins=50) ax[idxTTi][idxTTj].axvline(x=fisherThr, linestyle='--', color='pink') else: iBin = dfTrialTypeDict[pidType][iTT] >= fisherThr jBin = dfTrialTypeDict[pidType][jTT] >= fisherThr M = confusion_matrix(iBin, jBin) M = M.astype(float) / np.sum(M) # consistency = fisher_exact(M, alternative='two_sided')[0] # consistency = -np.log10(fisher_exact(M, alternative='two_sided')[1]) consistency = cohen_kappa_score(iBin, jBin) rezMat[idxTTi][idxTTj] = consistency sns.heatmap(ax=ax[idxTTi][idxTTj], data=M, annot=True, cmap='jet', xticklabels=fischerLabels, yticklabels=fischerLabels) else: # As consistency metric use correlation coefficient between values rezMat = np.zeros((nTrialTypes, nTrialTypes)) for idxTTi, iTT in enumerate(trialTypes): for idxTTj, jTT in enumerate(trialTypes): rezMat[idxTTi][idxTTj] = np.corrcoef( dfTrialTypeDict[pidType][iTT], dfTrialTypeDict[pidType][jTT])[0, 1] if kind == 'point': # As pairplot use scatter pPlot = sns.pairplot(data=dfTrialTypeDict[pidType], vars=trialTypes) #, kind='kde') elif kind == 'heatmap': # As pairplot use heatmap of binned scatter points fig, ax = plt.subplots(nrows=nTrialTypes, ncols=nTrialTypes, figsize=(4 * nTrialTypes, 4 * nTrialTypes)) for idxTTi, iTT in enumerate(trialTypes): ax[idxTTi, 0].set_ylabel(iTT) ax[-1, idxTTi].set_xlabel(iTT) for idxTTj, jTT in enumerate(trialTypes): if idxTTi == idxTTj: ax[idxTTi][idxTTj].hist( dfTrialTypeDict[pidType][iTT], range=[0, maxRange], bins=50) else: ax[idxTTi][idxTTj].hist2d( dfTrialTypeDict[pidType][iTT], dfTrialTypeDict[pidType][jTT], range=[[0, maxRange], [0, maxRange]], bins=[50, 50], cmap='jet') plt.savefig('pics/' + pidType + '_consistency_bymouse_scatter_' + fnameSuffix + '.png') plt.close() fig, ax = plt.subplots() sns.heatmap(ax=ax, data=rezMat, annot=True, cmap='jet', xticklabels=trialTypes, yticklabels=trialTypes, **limitKWargs) # imshow(fig, ax, rezMat, haveColorBar=True, limits=[0,1], xTicks=mice, yTicks=mice, cmap='jet') plt.savefig('pics/' + pidType + '_consistency_bymouse_metric_' + fnameSuffix + '.png') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2) dfConsistencyDict[pidType] = pd_append_row( dfConsistencyDict[pidType], [mousename, phase, avgConsistency]) for iPid, pidType in enumerate(pidTypes): fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistencyDict[pidType], *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, cmap='jet', **limitKWargs) fig.savefig('pics/' + 'summary_' + pidType + '_consistency_metric_' + datatype + '_' + str(performance) + '.png') plt.close()
def plot_consistency_significant_activity_byphase(dataDB, ds, intervals, minTrials=10, performance=None, dropChannels=None): rows = ds.list_dsets_pd() rows['mousename'] = [ dataDB.find_mouse_by_session(session) for session in rows['session'] ] dfColumns = ['datatype', 'trialType', 'consistency'] dfConsistency = pd.DataFrame(columns=dfColumns) for (datatype, trialType), rowsMouse in rows.groupby(['datatype', 'trialType']): pSigDict = {} for mousename, rowsSession in rowsMouse.groupby(['mousename']): pSig = [] for session, rowsTrial in rowsSession.groupby(['session']): if (performance is None) or dataDB.is_matching_performance( session, performance, mousename=mousename): assert intervals[0] in list(rowsTrial['intervName']) assert intervals[1] in list(rowsTrial['intervName']) dsetLabel1 = pd_is_one_row( pd_query(rowsTrial, {'intervName': intervals[0]}))[1]['dset'] dsetLabel2 = pd_is_one_row( pd_query(rowsTrial, {'intervName': intervals[1]}))[1]['dset'] data1 = ds.get_data(dsetLabel1) data2 = ds.get_data(dsetLabel2) nTrials1 = data1.shape[0] nTrials2 = data2.shape[1] if (nTrials1 < minTrials) or (nTrials2 < minTrials): print(session, datatype, trialType, 'too few trials', nTrials1, nTrials2, ';; skipping') else: nChannels = data1.shape[1] if dropChannels is not None: channelMask = np.ones(nChannels).astype(bool) channelMask[dropChannels] = 0 data1 = data1[:, channelMask] data2 = data2[:, channelMask] nChannels = nChannels - len(dropChannels) pvals = [ wilcoxon(data1[:, iCh], data2[:, iCh], alternative='two-sided')[1] for iCh in range(nChannels) ] # pSig += [(np.array(pvals) < 0.01).astype(int)] pSig += [-np.log10(np.array(pvals))] # pSigDict[mousename] = np.sum(pSig, axis=0) pSigDict[mousename] = np.mean(pSig, axis=0) mice = sorted(dataDB.mice) nMice = len(mice) corrCoef = np.zeros((nMice, nMice)) for iMouse, iName in enumerate(mice): for jMouse, jName in enumerate(mice): corrCoef[iMouse, jMouse] = np.corrcoef(pSigDict[iName], pSigDict[jName])[0, 1] sns.pairplot(data=pd.DataFrame(pSigDict), vars=mice) prefixPath = 'pics/consistency/significant_activity/byphase/bymouse/' make_path(prefixPath) plt.savefig(prefixPath + datatype + '_' + trialType + '.svg') plt.close() fig2, ax2 = plt.subplots() ax2.imshow(corrCoef, vmin=0, vmax=1) imshow(fig2, ax2, corrCoef, title='Significance Correlation', haveColorBar=True, limits=[0, 1], xTicks=mice, yTicks=mice) prefixPath = 'pics/consistency/significant_activity/byphase/bymouse_corr/' make_path(prefixPath) plt.savefig(prefixPath + datatype + '_' + trialType + '.svg') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(corrCoef)), 2) dfConsistency = pd_append_row(dfConsistency, [datatype, trialType, avgConsistency]) fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistency, *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, vmax=1, cmap='jet') prefixPath = 'pics/consistency/significant_activity/byphase/' make_path(prefixPath) fig.savefig(prefixPath + 'consistency_' + str(performance) + '.svg') plt.close()
def plot_consistency_significant_activity_byaction(dataDB, ds, minTrials=10, performance=None, dropChannels=None, metric='accuracy', limits=None): testFunc = test_metric_by_name(metric) rows = ds.list_dsets_pd() rows['mousename'] = [ dataDB.find_mouse_by_session(session) for session in rows['session'] ] dfColumns = ['datatype', 'phase', 'consistency'] dfConsistency = pd.DataFrame(columns=dfColumns) for (datatype, intervName), rowsMouse in rows.groupby(['datatype', 'intervName']): pSigDict = {} for mousename, rowsSession in rowsMouse.groupby(['mousename']): pSig = [] for session, rowsTrial in rowsSession.groupby(['session']): if (performance is None) or dataDB.is_matching_performance( session, performance, mousename=mousename): if len(rowsTrial) != 2: print(mousename, session, rowsTrial) raise ValueError('Expected exactly 2 rows') dsetLabels = list(rowsTrial['dset']) data1 = ds.get_data(dsetLabels[0]) data2 = ds.get_data(dsetLabels[1]) nTrials1 = data1.shape[0] nTrials2 = data2.shape[1] if (nTrials1 < minTrials) or (nTrials2 < minTrials): print(session, datatype, intervName, 'too few trials', nTrials1, nTrials2, ';; skipping') else: nChannels = data1.shape[1] if dropChannels is not None: channelMask = np.ones(nChannels).astype(bool) channelMask[dropChannels] = 0 data1 = data1[:, channelMask] data2 = data2[:, channelMask] nChannels = nChannels - len(dropChannels) pvals = [ testFunc(data1[:, iCh], data2[:, iCh]) for iCh in range(nChannels) ] # pSig += [(np.array(pvals) < 0.01).astype(int)] pSig += [-np.log10(np.array(pvals))] # pSigDict[mousename] = np.sum(pSig, axis=0) pSigDict[mousename] = np.mean(pSig, axis=0) mice = sorted(pSigDict.keys()) nMice = len(mice) corrCoef = np.zeros((nMice, nMice)) for iMouse, iName in enumerate(mice): for jMouse, jName in enumerate(mice): corrCoef[iMouse, jMouse] = np.corrcoef(pSigDict[iName], pSigDict[jName])[0, 1] plotSuffix = '_'.join([datatype, str(performance), intervName]) sns.pairplot(data=pd.DataFrame(pSigDict), vars=mice) prefixPath = 'pics/consistency/significant_activity/byaction/bymouse/' make_path(prefixPath) plt.savefig(prefixPath + plotSuffix + '.svg') plt.close() fig2, ax2 = plt.subplots() ax2.imshow(corrCoef, vmin=0, vmax=1) imshow(fig2, ax2, corrCoef, title='Significance Correlation', haveColorBar=True, limits=[0, 1], xTicks=mice, yTicks=mice) prefixPath = 'pics/consistency/significant_activity/byaction/bymouse_corr/' make_path(prefixPath) plt.savefig(prefixPath + plotSuffix + '.svg') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(corrCoef)), 2) dfConsistency = pd_append_row(dfConsistency, [datatype, intervName, avgConsistency]) fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistency, *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, vmax=1, cmap='jet') prefixPath = 'pics/consistency/significant_activity/byaction/' make_path(prefixPath) fig.savefig(prefixPath + 'consistency_' + str(performance) + '.svg') plt.close()
def barplot_conditions(ds, metricName, nameSuffix, verbose=True, trialTypes=None, intervNames=None): ''' Sweep over datatypes 1. (Mouse * [iGO, iNOGO]) @ {interv='AVG'} 2. (Mouse * interv / AVG) @ {trialType=None} ''' # 1. Extract all results for this test dfAll = ds.list_dsets_pd().fillna('None') dfAnalysis = pd_query(dfAll, {'metric' : metricName, "name" : nameSuffix}) dfAnalysis = pd_move_cols_front(dfAnalysis, ['metric', 'name', 'mousename']) # Move leading columns forwards for more informative printing/saving dfAnalysis = dfAnalysis.drop(['target_dim', 'datetime', 'shape'], axis=1) if 'performance' in dfAnalysis.columns: sweepLst = ['datatype', 'performance'] else: sweepLst = ['datatype'] for key, dfDataType in dfAnalysis.groupby(sweepLst): plotSuffix = '_'.join(key) if isinstance(key, list) else '_'.join([key]) if verbose: print(plotSuffix) intervNamesData = list(set(dfDataType['intervName'])) trialTypesData = list(set(dfDataType['trialType'])) intervNames = intervNamesData if intervNames is None else [i for i in intervNames if i in intervNamesData] trialTypes = trialTypesData if trialTypes is None else [i for i in trialTypes if i in trialTypesData] ################################# # Plot 1 ::: Mouse * TrialType ################################# for intervName in intervNames: df1 = pd_query(dfDataType, {'intervName' : intervName}) if trialTypes is not None: df1 = df1[df1['trialType'].isin(trialTypes)] dfData1 = pd.DataFrame(columns=['mousename', 'trialType', metricName]) for idx, row in df1.iterrows(): data = ds.get_data(row['dset']) for d in data: dfData1 = pd_append_row(dfData1, [row['mousename'], row['trialType'], d]) mice = sorted(set(dfData1['mousename'])) fig, ax = plt.subplots() sns_barplot(ax, dfData1, "mousename", metricName, 'trialType', annotHue=True, xOrd=mice, hOrd=trialTypes) # sns.barplot(ax=ax, x="mousename", y=metricName, hue='trialType', data=dfData1) prefixPath = 'pics/bulk/' + metricName + '/barplot_conditions/' make_path(prefixPath) fig.savefig(prefixPath + 'barplot_trialtype_' + plotSuffix + '_' + intervName + '.png', dpi=300) plt.close() ################################# # Plot 2 ::: Mouse * Phase ################################# for trialType in ['None'] + trialTypes: df2 = pd_query(dfDataType, {'trialType' : trialType}) # display(df2.head()) df2 = df2[df2['intervName'] != 'AVG'] if key[0] == 'bn_trial': df2 = df2[df2['intervName'] != 'PRE'] dfData2 = pd.DataFrame(columns=['mousename', 'phase', metricName]) for idx, row in df2.iterrows(): data = ds.get_data(row['dset']) for d in data: dfData2 = pd_append_row(dfData2, [row['mousename'], row['intervName'], d]) dfData2 = dfData2.sort_values('mousename') mice = sorted(set(dfData2['mousename'])) fig, ax = plt.subplots() sns_barplot(ax, dfData2, "mousename", metricName, 'phase', annotHue=False, xOrd=mice, hOrd=intervNames) # sns.barplot(ax=ax, x="mousename", y=metricName, hue='phase', data=dfData2) prefixPath = 'pics/bulk/' + metricName + '/barplot_conditions/' make_path(prefixPath) fig.savefig(prefixPath + 'barplot_phase_' + plotSuffix + '_' + trialType + '.png', dpi=300) plt.close()
def plot_pca_consistency(dataDB, intervNames=None, dropFirst=None, dropChannels=None): mice = sorted(dataDB.mice) nMice = len(mice) dfColumns = ['datatype', 'phase', 'consistency'] dfConsistency = pd.DataFrame(columns=dfColumns) if intervNames is None: intervNames = dataDB.get_interval_names() for datatype in dataDB.get_data_types(): for intervName in intervNames: fnameSuffix = datatype + '_' + intervName print(fnameSuffix) dataLst = [] for iMouse, mousename in enumerate(mice): dataRSPLst = dataDB.get_neuro_data({'mousename': mousename}, datatype=datatype, intervName=intervName) print(set([d.shape[1:] for d in dataRSPLst])) dataRSP = np.concatenate(dataRSPLst, axis=0) dataRP = np.mean(dataRSP, axis=1) dataRP = zscore(dataRP, axis=0) if dropChannels is not None: channelMask = np.ones(dataRP.shape[1]).astype(bool) channelMask[dropChannels] = 0 dataRP = dataRP[:, channelMask] dataLst += [dataRP] fig, ax = plt.subplots(nrows=nMice, ncols=nMice, figsize=(4 * nMice, 4 * nMice)) rezMat = np.zeros((nMice, nMice)) for iMouse in range(nMice): ax[iMouse][0].set_ylabel(mice[iMouse]) ax[-1][iMouse].set_xlabel(mice[iMouse]) for jMouse in range(nMice): rezXY, rezYX, arrXX, arrXY, arrYY, arrYX = pca.paired_comparison( dataLst[iMouse], dataLst[jMouse], dropFirst=dropFirst) rezMat[iMouse][jMouse] = rezXY # np.mean([rezXY, rezYX]) ax[iMouse][jMouse].plot(arrXX, label='XX') ax[iMouse][jMouse].plot(arrXY, label='XY') ax[iMouse][jMouse].legend() plt.savefig('pca_consistency_bymouse_evals_' + fnameSuffix + '.svg') plt.close() fig, ax = plt.subplots() imshow(fig, ax, rezMat, haveColorBar=True, limits=[0, 1], xTicks=mice, yTicks=mice) plt.savefig('pca_consistency_bymouse_metric_' + fnameSuffix + '.svg') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2) dfConsistency = pd_append_row( dfConsistency, [datatype, intervName, avgConsistency]) fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistency, *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, vmin=0, vmax=1, cmap='jet') fig.savefig('consistency_coactivity_metric.svg') plt.close()
def plot_corr_consistency_l1_phase(dataDB, nDropPCA=None, dropChannels=None, performance=None, datatype=None): mice = sorted(dataDB.mice) phases = dataDB.get_interval_names() nPhases = len(phases) dfColumns = ['mousename', 'trialtype', 'consistency'] dfConsistency = pd.DataFrame(columns=dfColumns) for iMouse, mousename in enumerate(mice): for trialType in dataDB.get_trial_type_names(): fnameSuffix = '_'.join( [datatype, mousename, trialType, str(performance)]) print(fnameSuffix) corrLst = [] for intervName in phases: kwargs = { 'datatype': datatype, 'intervName': intervName, 'trialType': trialType } if performance is not None: kwargs['performance'] = performance dataRSPLst = dataDB.get_neuro_data({'mousename': mousename}, zscoreDim='rs', **kwargs) dataRSP = np.concatenate(dataRSPLst, axis=0) dataRP = np.mean(dataRSP, axis=1) # dataRP = zscore(dataRP, axis=0) if dropChannels is not None: channelMask = np.ones(dataRP.shape[1]).astype(bool) channelMask[dropChannels] = 0 dataRP = dataRP[:, channelMask] if nDropPCA is not None: dataRP = drop_PCA(dataRP, nDropPCA) corrLst += [tril_1D(np.corrcoef(dataRP.T))] # fig, ax = plt.subplots(nrows=nMice, ncols=nMice, figsize=(4 * nMice, 4 * nMice)) pairDict = {} rezMat = np.zeros((nPhases, nPhases)) for idxNamei, iName in enumerate(phases): pairDict[iName] = corrLst[idxNamei] for idxNamej, jName in enumerate(phases): rezMat[idxNamei][idxNamej] = np.corrcoef( corrLst[idxNamei], corrLst[idxNamej])[0, 1] pPlot = sns.pairplot(data=pd.DataFrame(pairDict), vars=phases, kind='kde') prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str( nDropPCA) + '/scatter/' make_path(prefixPath) plt.savefig(prefixPath + 'scatter_' + fnameSuffix + '.svg') plt.close() fig, ax = plt.subplots() imshow(fig, ax, rezMat, haveColorBar=True, limits=[0, 1], xTicks=phases, yTicks=phases, cmap='jet') prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str( nDropPCA) + '/metric/' make_path(prefixPath) plt.savefig(prefixPath + 'metric_' + fnameSuffix + '.svg') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2) dfConsistency = pd_append_row( dfConsistency, [mousename, trialType, avgConsistency]) fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistency, *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, vmin=0, vmax=1, cmap='jet') prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str(nDropPCA) + '/' make_path(prefixPath) fig.savefig(prefixPath + datatype + '_' + str(performance) + '.svg') plt.close()
def plot_corr_consistency_l1_mouse( dataDB, nDropPCA=None, dropChannels=None, exclQueryLst=None, **kwargs): # performances=None, trialTypes=None, assert 'intervName' in kwargs.keys(), 'Requires phases' dps = DataParameterSweep(dataDB, exclQueryLst, mousename='auto', intervName='auto', datatype='auto', **kwargs) mice = sorted(dataDB.mice) nMice = len(mice) for paramExtraVals, dfTmp in dps.sweepDF.groupby( dps.invert_param(['mousename', 'datatype', 'intervName'])): plotExtraSuffix = param_vals_to_suffix(paramExtraVals) dfColumns = ['datatype', 'phase', 'consistency'] dfConsistency = pd.DataFrame(columns=dfColumns) for paramVals, dfMouse in dfTmp.groupby(dps.invert_param(['mousename' ])): plotSuffix = param_vals_to_suffix(paramVals) print(plotSuffix) corrLst = [] for idx, row in dfMouse.iterrows(): plotSuffix = '_'.join([str(s) for s in row.values]) print(plotSuffix) kwargsThis = pd_row_to_kwargs(row, parseNone=True, dropKeys=['mousename']) dataRSPLst = dataDB.get_neuro_data( { 'mousename': row['mousename'] }, # NOTE: zscore channels for each session to avoid session-wise effects zscoreDim='rs', **kwargsThis) dataRSP = np.concatenate(dataRSPLst, axis=0) dataRP = np.mean(dataRSP, axis=1) # dataRP = zscore(dataRP, axis=0) if dropChannels is not None: channelMask = np.ones(dataRP.shape[1]).astype(bool) channelMask[dropChannels] = 0 dataRP = dataRP[:, channelMask] if nDropPCA is not None: dataRP = drop_PCA(dataRP, nDropPCA) corrLst += [tril_1D(np.corrcoef(dataRP.T))] # fig, ax = plt.subplots(nrows=nMice, ncols=nMice, figsize=(4 * nMice, 4 * nMice)) pairDict = {} rezMat = np.zeros((nMice, nMice)) for iMouse, mousename in enumerate(mice): # ax[iMouse][0].set_ylabel(mice[iMouse]) # ax[-1][iMouse].set_xlabel(mice[iMouse]) pairDict[mousename] = corrLst[iMouse] for jMouse in range(nMice): # rezMat[iMouse][jMouse] = 1 - rmae(corrLst[iMouse], corrLst[jMouse]) rezMat[iMouse][jMouse] = np.corrcoef( corrLst[iMouse], corrLst[jMouse])[0, 1] # cci = offdiag_1D(corrLst[iMouse]) # ccj = offdiag_1D(corrLst[jMouse]) # ax[iMouse][jMouse].plot(cci, ccj, '.') pPlot = sns.pairplot(data=pd.DataFrame(pairDict), vars=mice, kind='kde') prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str( nDropPCA) + '/scatter/' make_path(prefixPath) plt.savefig(prefixPath + 'scatter_' + plotSuffix + '.svg') plt.close() fig, ax = plt.subplots() imshow(fig, ax, rezMat, haveColorBar=True, limits=[0, 1], xTicks=mice, yTicks=mice, cmap='jet') prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str( nDropPCA) + '/metric/' make_path(prefixPath) plt.savefig(prefixPath + 'metric_' + plotSuffix + '.svg') plt.close() avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2) dfConsistency = pd_append_row( dfConsistency, [row['datatype'], row['intervName'], avgConsistency]) fig, ax = plt.subplots() dfPivot = pd_pivot(dfConsistency, *dfColumns) sns.heatmap(data=dfPivot, ax=ax, annot=True, vmin=0, vmax=1, cmap='jet') prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str( nDropPCA) + '/' make_path(prefixPath) fig.savefig(prefixPath + plotExtraSuffix + '.svg') plt.close()
def find_parse_tdt(self, pathTDT): print('Parsing TDT:') for mouseName, dfMouse in self.dfSession.groupby(['mousename']): pathMouse = os.path.join(pathTDT, mouseName) prepcommon._testdir(pathMouse) for dayName, dfDay in dfMouse.groupby(['dateKey']): print('--', mouseName, dayName) pathDay = os.path.join(pathMouse, 'TDT', dayName, 'widefield_labview') pathAllen = os.path.join(pathDay, 'ROI_Allen', 'registration_transform_1510.mat') prepcommon._testdir(pathDay) prepcommon._testfile(pathAllen) self.pathT1[mouseName][dayName] = pathAllen for idx, row in dfDay.iterrows(): sessionSuffix = row['sessionKey'] pathSession = os.path.join(pathDay, sessionSuffix) pathMat = os.path.join(pathSession, 'Matt_files') pathTrialInd = os.path.join(pathMat, 'trials_ind.mat') if mouseName == 'mou_5': matFiles = [f for f in os.listdir(pathMat) if os.path.isfile(os.path.join(pathMat, f))] matFiles = [f for f in matFiles if '.mat' in f] matFiles = [f for f in matFiles if dayName[:4] in f] assert len(matFiles) == 1 pathTrialStructMat = os.path.join(pathMat, matFiles[0]) else: pathTrialStructMat = os.path.join(pathMat, dayName + sessionSuffix + '.mat') if mouseName == 'mou_5': sessionKeyLabview = '_s' + sessionSuffix[-1] elif mouseName == 'mou_6': sessionKeyLabview = '_s' + str(self._letters_to_sessions(sessionSuffix)) else: sessionKeyLabview = '_' + sessionSuffix pathTrialStructLabview = os.path.join(pathSession, mouseName + sessionKeyLabview + ''.join(dayName.split('_')) + '.txt') pathActivePassive = os.path.join(pathMat, 'trials_with_and_wo_initial_moves_OCIA_from_movie.mat') pathMovementVectors = os.path.join(pathMat, 'move_vectors_from_movie.mat') prepcommon._testdir(pathSession) prepcommon._testdir(pathMat) prepcommon._testfile(pathTrialInd) prepcommon._testfile(pathTrialStructMat) prepcommon._testfile(pathTrialStructLabview, critical=False) prepcommon._testfile(pathActivePassive, critical=False) prepcommon._testfile(pathMovementVectors, critical=False) self.dataPaths = pd_append_row(self.dataPaths, [ mouseName, dayName, sessionSuffix, pathSession, pathTrialInd, pathTrialStructMat, pathTrialStructLabview, pathActivePassive, pathMovementVectors ])