def find_parse_data_paths(self, pathAUD):
for mouseName, dfMouse in self.dfSession.groupby(['mousename']):
pathMouse = os.path.join(pathAUD, mouseName)
prepcommon._testdir(pathMouse)
for dayName, dfDay in dfMouse.groupby(['dateKey']):
print(mouseName, dayName)
pathDay = os.path.join(pathMouse, dayName, 'widefield_labview')
pathAllen = os.path.join(pathDay, 'ROI_Allen',
'registration_transform_1510.mat')
haveDir = prepcommon._testdir(pathDay, critical=False)
haveFile = prepcommon._testfile(pathAllen, critical=False)
if haveDir and haveFile:
self.pathT1[mouseName][dayName] = pathAllen
for idx, row in dfDay.iterrows():
sessionSuffix = row['sessionKey']
pathSession = os.path.join(pathDay, sessionSuffix)
pathMat = os.path.join(pathSession, 'Matt_files')
# pathTrialInd = os.path.join(pathMat, 'trials_ind.mat')
matFiles = [
f for f in os.listdir(pathMat)
if os.path.splitext(f)[1] == '.mat'
and f[:9] == 'Behavior_'
]
assert len(matFiles) == 1
pathTrialStruct = os.path.join(pathMat, matFiles[0])
pathActivePassive = os.path.join(
pathMat,
'trials_with_and_wo_initial_moves_OCIA_from_movie.mat'
)
pathMovementVectors = os.path.join(
pathMat, 'move_vectors_from_movie.mat')
prepcommon._testdir(pathSession)
prepcommon._testdir(pathMat)
# prepcommon._testfile(pathTrialInd)
prepcommon._testfile(pathTrialStruct)
prepcommon._testfile(pathActivePassive)
prepcommon._testfile(pathMovementVectors,
critical=False)
self.dataPaths = pd_append_row(
self.dataPaths,
[
mouseName,
dayName,
sessionSuffix,
pathSession, #pathTrialInd,
pathTrialStruct,
pathActivePassive,
pathMovementVectors
])
def process_metadata_files(self, pwd):
for mouseName, mouseRows in self.dataPaths.groupby(['mouse']):
h5name = os.path.join(pwd, mouseName + '.h5')
prepcommon._h5_append_group(h5name, 'metadataTrial')
prepcommon._h5_append_group(h5name, 'accuracy')
prepcommon._h5_append_group(h5name, 'dprime')
dfMetaSession = pd.DataFrame(columns=['session', 'timestamp', 'delay'])
for idx, row in mouseRows.iterrows():
sessionName = row['day'] + '_' + row['session']
with h5py.File(h5name, 'a') as h5f:
metaProcessed = sessionName in h5f['metadataTrial'].keys()
print(mouseName, sessionName)
if metaProcessed:
print('-- already processed')
continue
# Read metadata from two file types and data, reconcile
with h5py.File(h5name, 'r') as f:
dataRSPShape = f['data'][sessionName].shape
dfTrialStructMat = self.read_trial_structure_as_pd(row['trialStructPathMat'], mouseName)
timeStamp, dfTrialStructLabview = self.read_parse_labview_trial_structure_as_pd(row['trialStructPathLabview'])
delay, dfTrialStruct = self.reconcile_trial_structure(dfTrialStructMat, dfTrialStructLabview, dataRSPShape)
# Store to file
dfTrialStruct.to_hdf(h5name, '/metadataTrial/' + sessionName)
dfMetaSession = pd_append_row(dfMetaSession, [sessionName, timeStamp, delay])
# Calculate and store accuracy and dprime
ttDict = prepcommon.count_trial_types(dfTrialStruct)
print(ttDict)
acc = accuracy(ttDict['Hit'], ttDict['Miss'], ttDict['FA'], ttDict['CR'])
dp = d_prime(ttDict['Hit'], ttDict['Miss'], ttDict['FA'], ttDict['CR'])
with h5py.File(h5name, 'a') as h5f:
h5f['accuracy'].create_dataset(sessionName, data=acc)
h5f['dprime'].create_dataset(sessionName, data=dp)
dfMetaSession.to_hdf(h5name, 'metadataSession')
def extract_timestamps_video(self, pwd):
for mouseName, dfMouse in self.dataPaths.groupby(['mouse']):
h5name = os.path.join(pwd, mouseName + '.h5')
for idx, row in dfMouse.iterrows():
sessionName = row['day'] + '_' + row['session']
print(mouseName, sessionName)
filePaths = self.parse_video_paths(row['sessionPath'],
row['day'])
fileBases = [os.path.basename(f) for f in filePaths]
timeStampStrings = [
':'.join(
[f[:4], f[4:6], f[6:8], f[9:11], f[11:13], f[13:15]])
for f in fileBases
]
timeStamps = [
datetime.strptime(f, "%Y:%d:%m:%H:%M:%S")
for f in timeStampStrings
]
metadata = pd.read_hdf(h5name, '/metadata/' + sessionName)
if len(timeStamps) != len(metadata):
print(mouseName, sessionName, len(timeStamps),
len(metadata))
if len(timeStamps) < len(metadata):
timeStamps += [None] * (len(metadata) - len(timeStamps))
elif len(timeStamps) == len(metadata) + 1:
metadata = pd_append_row(metadata,
[None] * len(metadata.columns))
print(metadata.tail())
metadata['timeStamps'] = timeStamps
metadata.to_hdf(h5name, '/metadata/' + sessionName)
def plot_all_frac_significant_performance_scatter(dataDB,
h5fname,
minTrials=50):
pidTypes = ['unique', 'red', 'syn']
summaryDF = pid_all_summary_df(h5fname)
for keyLst, dfSession in summaryDF.groupby(['datatype',
'phase']): # 'mousename'
keyLabel = '_'.join(keyLst)
pidDFAll = pd.DataFrame(columns=['pid', 'x', 'y', 'mouse'])
pidDictNaive = defaultdict(list)
pidDictExpert = defaultdict(list)
fig, ax = plt.subplots(nrows=2,
ncols=len(pidTypes),
figsize=(len(pidTypes) * 4, 8),
tight_layout=True)
for keyLst2, dfSession2 in dfSession.groupby(['mousename'
]): # 'mousename'
print(keyLabel, keyLst2)
for idx, row in dfSession2.iterrows():
nTrialsThis = dataDB.get_ntrial_bytype(
{'session': dfSession['session']}, trialType='iGO')
if nTrialsThis < minTrials:
print('Skipping session', dfSession['session'],
'because it has too few trials', nTrialsThis)
else:
perf = dataDB.get_performance(row['session'])
df1 = pd.read_hdf(h5fname, key=row['key'])
# Merge Unique1 and Unique2 for this plot
df1 = preprocess_unique(df1)
for pidType in pidTypes:
pVal = df1[df1['PID'] == pidType]['p']
fracSig = np.mean(pVal < 0.01)
pidDFAll = pd_append_row(
pidDFAll, [pidType, perf, fracSig, keyLst2])
# pidDFAll[pidType] += [(perf, fracSig, keyLst2)]
if perf < 0.7:
pidDictNaive[pidType] += [fracSig]
else:
pidDictExpert[pidType] += [fracSig]
for iFig, pidType in enumerate(pidTypes):
ax[0, iFig].set_title(pidType)
ax[0, iFig].set_xlabel('Performance')
ax[0, iFig].set_ylabel('Fraction Significant')
dataThis = pidDFAll[pidDFAll['pid'] == pidType]
sns.scatterplot(ax=ax[0, iFig],
data=dataThis,
x='x',
y='y',
hue='mouse')
sns.regplot(ax=ax[0, iFig],
data=dataThis,
x='x',
y='y',
scatter=False)
ax[0, iFig].set_ylim(-0.01, 1.01)
# sns.regplot(x=x, y=y, ax=ax[0, iFig])
# ax[0, iFig].plot(*np.array(pidDictSession[pidType]).T, '.', label=str(keyLst2))
for iFig, pidType in enumerate(pidTypes):
ax[0, iFig].legend()
ax[0, iFig].set_xlim([0, 1])
ax[0, iFig].axvline(x=0.7, linestyle='--', color='gray', alpha=0.5)
violins_labeled(ax[1, iFig],
[pidDictNaive[pidType], pidDictExpert[pidType]],
['naive', 'expert'],
'performance',
'Fraction Significant',
violinInner='box',
style='bar',
sigTestPairs=[[0, 1]])
ax[1, iFig].set_ylim([0, 1])
# plt.show()
plt.savefig('PID_Freq_Significant_vs_perf_' + keyLabel + '.png')
plt.close()
def plot_consistency_bytrialtype(h5fname,
dfSummary,
dropChannels=None,
performance=None,
datatype=None,
trialTypes=None,
kind='point',
fisherThr=0.1,
limits=None):
pidTypes = ['unique', 'syn', 'red']
limitKWargs = {
'vmin': limits[0],
'vmax': limits[1]
} if limits is not None else {}
if performance is None:
dfSummaryEff = pd_query(dfSummary, {'datatype': datatype})
else:
dfSummaryEff = pd_query(dfSummary, {
'datatype': datatype,
'performance': performance
})
dfColumns = ['mousename', 'phase', 'consistency']
dfConsistencyDict = {
pidType: pd.DataFrame(columns=dfColumns)
for pidType in pidTypes
}
for (mousename, phase), df1 in dfSummaryEff.groupby(['mousename',
'phase']):
fnameSuffix = '_'.join([mousename, datatype, phase, str(performance)])
trialTypes = trialTypes if trialTypes is not None else sorted(
list(set(df1['trialType'])))
nTrialTypes = len(trialTypes)
dfTrialTypeDict = {}
for iPid, pidType in enumerate(pidTypes):
dfTrialTypeDict[pidType] = pd.DataFrame()
for trialType, dfTrialType in df1.groupby(['trialType']):
if trialType in trialTypes:
dfTmp = read_parse_joint_dataframe(dfTrialType,
h5fname, [mousename],
pidTypes,
dropChannels=dropChannels)
for iPid, pidType in enumerate(pidTypes):
dfTrialTypeDict[pidType][trialType] = dfTmp[pidType][
'muTrue_' + mousename]
for iPid, pidType in enumerate(pidTypes):
maxRange = 0.35 if pidType == 'syn' else 1.0
# As consistency metric perform Fischer's exact test for significant vs unsignificant links
# As pairplot show contingency tables
if kind == 'fisher':
fischerLabels = ['low', 'high']
rezMat = np.full((nTrialTypes, nTrialTypes), np.nan)
fig, ax = plt.subplots(nrows=nTrialTypes,
ncols=nTrialTypes,
figsize=(4 * nTrialTypes,
4 * nTrialTypes))
for idxTTi, iTT in enumerate(trialTypes):
ax[idxTTi, 0].set_ylabel(iTT)
ax[-1, idxTTi].set_xlabel(iTT)
for idxTTj, jTT in enumerate(trialTypes):
if idxTTi == idxTTj:
ax[idxTTi][idxTTj].hist(
dfTrialTypeDict[pidType][iTT],
range=[0, maxRange],
bins=50)
ax[idxTTi][idxTTj].axvline(x=fisherThr,
linestyle='--',
color='pink')
else:
iBin = dfTrialTypeDict[pidType][iTT] >= fisherThr
jBin = dfTrialTypeDict[pidType][jTT] >= fisherThr
M = confusion_matrix(iBin, jBin)
M = M.astype(float) / np.sum(M)
# consistency = fisher_exact(M, alternative='two_sided')[0]
# consistency = -np.log10(fisher_exact(M, alternative='two_sided')[1])
consistency = cohen_kappa_score(iBin, jBin)
rezMat[idxTTi][idxTTj] = consistency
sns.heatmap(ax=ax[idxTTi][idxTTj],
data=M,
annot=True,
cmap='jet',
xticklabels=fischerLabels,
yticklabels=fischerLabels)
else:
# As consistency metric use correlation coefficient between values
rezMat = np.zeros((nTrialTypes, nTrialTypes))
for idxTTi, iTT in enumerate(trialTypes):
for idxTTj, jTT in enumerate(trialTypes):
rezMat[idxTTi][idxTTj] = np.corrcoef(
dfTrialTypeDict[pidType][iTT],
dfTrialTypeDict[pidType][jTT])[0, 1]
if kind == 'point':
# As pairplot use scatter
pPlot = sns.pairplot(data=dfTrialTypeDict[pidType],
vars=trialTypes) #, kind='kde')
elif kind == 'heatmap':
# As pairplot use heatmap of binned scatter points
fig, ax = plt.subplots(nrows=nTrialTypes,
ncols=nTrialTypes,
figsize=(4 * nTrialTypes,
4 * nTrialTypes))
for idxTTi, iTT in enumerate(trialTypes):
ax[idxTTi, 0].set_ylabel(iTT)
ax[-1, idxTTi].set_xlabel(iTT)
for idxTTj, jTT in enumerate(trialTypes):
if idxTTi == idxTTj:
ax[idxTTi][idxTTj].hist(
dfTrialTypeDict[pidType][iTT],
range=[0, maxRange],
bins=50)
else:
ax[idxTTi][idxTTj].hist2d(
dfTrialTypeDict[pidType][iTT],
dfTrialTypeDict[pidType][jTT],
range=[[0, maxRange], [0, maxRange]],
bins=[50, 50],
cmap='jet')
plt.savefig('pics/' + pidType + '_consistency_bymouse_scatter_' +
fnameSuffix + '.png')
plt.close()
fig, ax = plt.subplots()
sns.heatmap(ax=ax,
data=rezMat,
annot=True,
cmap='jet',
xticklabels=trialTypes,
yticklabels=trialTypes,
**limitKWargs)
# imshow(fig, ax, rezMat, haveColorBar=True, limits=[0,1], xTicks=mice, yTicks=mice, cmap='jet')
plt.savefig('pics/' + pidType + '_consistency_bymouse_metric_' +
fnameSuffix + '.png')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2)
dfConsistencyDict[pidType] = pd_append_row(
dfConsistencyDict[pidType], [mousename, phase, avgConsistency])
for iPid, pidType in enumerate(pidTypes):
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistencyDict[pidType], *dfColumns)
sns.heatmap(data=dfPivot, ax=ax, annot=True, cmap='jet', **limitKWargs)
fig.savefig('pics/' + 'summary_' + pidType + '_consistency_metric_' +
datatype + '_' + str(performance) + '.png')
plt.close()
def plot_consistency_significant_activity_byphase(dataDB,
ds,
intervals,
minTrials=10,
performance=None,
dropChannels=None):
rows = ds.list_dsets_pd()
rows['mousename'] = [
dataDB.find_mouse_by_session(session) for session in rows['session']
]
dfColumns = ['datatype', 'trialType', 'consistency']
dfConsistency = pd.DataFrame(columns=dfColumns)
for (datatype,
trialType), rowsMouse in rows.groupby(['datatype', 'trialType']):
pSigDict = {}
for mousename, rowsSession in rowsMouse.groupby(['mousename']):
pSig = []
for session, rowsTrial in rowsSession.groupby(['session']):
if (performance is None) or dataDB.is_matching_performance(
session, performance, mousename=mousename):
assert intervals[0] in list(rowsTrial['intervName'])
assert intervals[1] in list(rowsTrial['intervName'])
dsetLabel1 = pd_is_one_row(
pd_query(rowsTrial,
{'intervName': intervals[0]}))[1]['dset']
dsetLabel2 = pd_is_one_row(
pd_query(rowsTrial,
{'intervName': intervals[1]}))[1]['dset']
data1 = ds.get_data(dsetLabel1)
data2 = ds.get_data(dsetLabel2)
nTrials1 = data1.shape[0]
nTrials2 = data2.shape[1]
if (nTrials1 < minTrials) or (nTrials2 < minTrials):
print(session, datatype, trialType, 'too few trials',
nTrials1, nTrials2, ';; skipping')
else:
nChannels = data1.shape[1]
if dropChannels is not None:
channelMask = np.ones(nChannels).astype(bool)
channelMask[dropChannels] = 0
data1 = data1[:, channelMask]
data2 = data2[:, channelMask]
nChannels = nChannels - len(dropChannels)
pvals = [
wilcoxon(data1[:, iCh],
data2[:, iCh],
alternative='two-sided')[1]
for iCh in range(nChannels)
]
# pSig += [(np.array(pvals) < 0.01).astype(int)]
pSig += [-np.log10(np.array(pvals))]
# pSigDict[mousename] = np.sum(pSig, axis=0)
pSigDict[mousename] = np.mean(pSig, axis=0)
mice = sorted(dataDB.mice)
nMice = len(mice)
corrCoef = np.zeros((nMice, nMice))
for iMouse, iName in enumerate(mice):
for jMouse, jName in enumerate(mice):
corrCoef[iMouse, jMouse] = np.corrcoef(pSigDict[iName],
pSigDict[jName])[0, 1]
sns.pairplot(data=pd.DataFrame(pSigDict), vars=mice)
prefixPath = 'pics/consistency/significant_activity/byphase/bymouse/'
make_path(prefixPath)
plt.savefig(prefixPath + datatype + '_' + trialType + '.svg')
plt.close()
fig2, ax2 = plt.subplots()
ax2.imshow(corrCoef, vmin=0, vmax=1)
imshow(fig2,
ax2,
corrCoef,
title='Significance Correlation',
haveColorBar=True,
limits=[0, 1],
xTicks=mice,
yTicks=mice)
prefixPath = 'pics/consistency/significant_activity/byphase/bymouse_corr/'
make_path(prefixPath)
plt.savefig(prefixPath + datatype + '_' + trialType + '.svg')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(corrCoef)), 2)
dfConsistency = pd_append_row(dfConsistency,
[datatype, trialType, avgConsistency])
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistency, *dfColumns)
sns.heatmap(data=dfPivot, ax=ax, annot=True, vmax=1, cmap='jet')
prefixPath = 'pics/consistency/significant_activity/byphase/'
make_path(prefixPath)
fig.savefig(prefixPath + 'consistency_' + str(performance) + '.svg')
plt.close()
def plot_consistency_significant_activity_byaction(dataDB,
ds,
minTrials=10,
performance=None,
dropChannels=None,
metric='accuracy',
limits=None):
testFunc = test_metric_by_name(metric)
rows = ds.list_dsets_pd()
rows['mousename'] = [
dataDB.find_mouse_by_session(session) for session in rows['session']
]
dfColumns = ['datatype', 'phase', 'consistency']
dfConsistency = pd.DataFrame(columns=dfColumns)
for (datatype,
intervName), rowsMouse in rows.groupby(['datatype', 'intervName']):
pSigDict = {}
for mousename, rowsSession in rowsMouse.groupby(['mousename']):
pSig = []
for session, rowsTrial in rowsSession.groupby(['session']):
if (performance is None) or dataDB.is_matching_performance(
session, performance, mousename=mousename):
if len(rowsTrial) != 2:
print(mousename, session, rowsTrial)
raise ValueError('Expected exactly 2 rows')
dsetLabels = list(rowsTrial['dset'])
data1 = ds.get_data(dsetLabels[0])
data2 = ds.get_data(dsetLabels[1])
nTrials1 = data1.shape[0]
nTrials2 = data2.shape[1]
if (nTrials1 < minTrials) or (nTrials2 < minTrials):
print(session, datatype, intervName, 'too few trials',
nTrials1, nTrials2, ';; skipping')
else:
nChannels = data1.shape[1]
if dropChannels is not None:
channelMask = np.ones(nChannels).astype(bool)
channelMask[dropChannels] = 0
data1 = data1[:, channelMask]
data2 = data2[:, channelMask]
nChannels = nChannels - len(dropChannels)
pvals = [
testFunc(data1[:, iCh], data2[:, iCh])
for iCh in range(nChannels)
]
# pSig += [(np.array(pvals) < 0.01).astype(int)]
pSig += [-np.log10(np.array(pvals))]
# pSigDict[mousename] = np.sum(pSig, axis=0)
pSigDict[mousename] = np.mean(pSig, axis=0)
mice = sorted(pSigDict.keys())
nMice = len(mice)
corrCoef = np.zeros((nMice, nMice))
for iMouse, iName in enumerate(mice):
for jMouse, jName in enumerate(mice):
corrCoef[iMouse, jMouse] = np.corrcoef(pSigDict[iName],
pSigDict[jName])[0, 1]
plotSuffix = '_'.join([datatype, str(performance), intervName])
sns.pairplot(data=pd.DataFrame(pSigDict), vars=mice)
prefixPath = 'pics/consistency/significant_activity/byaction/bymouse/'
make_path(prefixPath)
plt.savefig(prefixPath + plotSuffix + '.svg')
plt.close()
fig2, ax2 = plt.subplots()
ax2.imshow(corrCoef, vmin=0, vmax=1)
imshow(fig2,
ax2,
corrCoef,
title='Significance Correlation',
haveColorBar=True,
limits=[0, 1],
xTicks=mice,
yTicks=mice)
prefixPath = 'pics/consistency/significant_activity/byaction/bymouse_corr/'
make_path(prefixPath)
plt.savefig(prefixPath + plotSuffix + '.svg')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(corrCoef)), 2)
dfConsistency = pd_append_row(dfConsistency,
[datatype, intervName, avgConsistency])
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistency, *dfColumns)
sns.heatmap(data=dfPivot, ax=ax, annot=True, vmax=1, cmap='jet')
prefixPath = 'pics/consistency/significant_activity/byaction/'
make_path(prefixPath)
fig.savefig(prefixPath + 'consistency_' + str(performance) + '.svg')
plt.close()
def barplot_conditions(ds, metricName, nameSuffix, verbose=True, trialTypes=None, intervNames=None):
'''
Sweep over datatypes
1. (Mouse * [iGO, iNOGO]) @ {interv='AVG'}
2. (Mouse * interv / AVG) @ {trialType=None}
'''
# 1. Extract all results for this test
dfAll = ds.list_dsets_pd().fillna('None')
dfAnalysis = pd_query(dfAll, {'metric' : metricName, "name" : nameSuffix})
dfAnalysis = pd_move_cols_front(dfAnalysis, ['metric', 'name', 'mousename']) # Move leading columns forwards for more informative printing/saving
dfAnalysis = dfAnalysis.drop(['target_dim', 'datetime', 'shape'], axis=1)
if 'performance' in dfAnalysis.columns:
sweepLst = ['datatype', 'performance']
else:
sweepLst = ['datatype']
for key, dfDataType in dfAnalysis.groupby(sweepLst):
plotSuffix = '_'.join(key) if isinstance(key, list) else '_'.join([key])
if verbose:
print(plotSuffix)
intervNamesData = list(set(dfDataType['intervName']))
trialTypesData = list(set(dfDataType['trialType']))
intervNames = intervNamesData if intervNames is None else [i for i in intervNames if i in intervNamesData]
trialTypes = trialTypesData if trialTypes is None else [i for i in trialTypes if i in trialTypesData]
#################################
# Plot 1 ::: Mouse * TrialType
#################################
for intervName in intervNames:
df1 = pd_query(dfDataType, {'intervName' : intervName})
if trialTypes is not None:
df1 = df1[df1['trialType'].isin(trialTypes)]
dfData1 = pd.DataFrame(columns=['mousename', 'trialType', metricName])
for idx, row in df1.iterrows():
data = ds.get_data(row['dset'])
for d in data:
dfData1 = pd_append_row(dfData1, [row['mousename'], row['trialType'], d])
mice = sorted(set(dfData1['mousename']))
fig, ax = plt.subplots()
sns_barplot(ax, dfData1, "mousename", metricName, 'trialType', annotHue=True, xOrd=mice, hOrd=trialTypes)
# sns.barplot(ax=ax, x="mousename", y=metricName, hue='trialType', data=dfData1)
prefixPath = 'pics/bulk/' + metricName + '/barplot_conditions/'
make_path(prefixPath)
fig.savefig(prefixPath + 'barplot_trialtype_' + plotSuffix + '_' + intervName + '.png', dpi=300)
plt.close()
#################################
# Plot 2 ::: Mouse * Phase
#################################
for trialType in ['None'] + trialTypes:
df2 = pd_query(dfDataType, {'trialType' : trialType})
# display(df2.head())
df2 = df2[df2['intervName'] != 'AVG']
if key[0] == 'bn_trial':
df2 = df2[df2['intervName'] != 'PRE']
dfData2 = pd.DataFrame(columns=['mousename', 'phase', metricName])
for idx, row in df2.iterrows():
data = ds.get_data(row['dset'])
for d in data:
dfData2 = pd_append_row(dfData2, [row['mousename'], row['intervName'], d])
dfData2 = dfData2.sort_values('mousename')
mice = sorted(set(dfData2['mousename']))
fig, ax = plt.subplots()
sns_barplot(ax, dfData2, "mousename", metricName, 'phase', annotHue=False, xOrd=mice, hOrd=intervNames)
# sns.barplot(ax=ax, x="mousename", y=metricName, hue='phase', data=dfData2)
prefixPath = 'pics/bulk/' + metricName + '/barplot_conditions/'
make_path(prefixPath)
fig.savefig(prefixPath + 'barplot_phase_' + plotSuffix + '_' + trialType + '.png', dpi=300)
plt.close()
def plot_pca_consistency(dataDB,
intervNames=None,
dropFirst=None,
dropChannels=None):
mice = sorted(dataDB.mice)
nMice = len(mice)
dfColumns = ['datatype', 'phase', 'consistency']
dfConsistency = pd.DataFrame(columns=dfColumns)
if intervNames is None:
intervNames = dataDB.get_interval_names()
for datatype in dataDB.get_data_types():
for intervName in intervNames:
fnameSuffix = datatype + '_' + intervName
print(fnameSuffix)
dataLst = []
for iMouse, mousename in enumerate(mice):
dataRSPLst = dataDB.get_neuro_data({'mousename': mousename},
datatype=datatype,
intervName=intervName)
print(set([d.shape[1:] for d in dataRSPLst]))
dataRSP = np.concatenate(dataRSPLst, axis=0)
dataRP = np.mean(dataRSP, axis=1)
dataRP = zscore(dataRP, axis=0)
if dropChannels is not None:
channelMask = np.ones(dataRP.shape[1]).astype(bool)
channelMask[dropChannels] = 0
dataRP = dataRP[:, channelMask]
dataLst += [dataRP]
fig, ax = plt.subplots(nrows=nMice,
ncols=nMice,
figsize=(4 * nMice, 4 * nMice))
rezMat = np.zeros((nMice, nMice))
for iMouse in range(nMice):
ax[iMouse][0].set_ylabel(mice[iMouse])
ax[-1][iMouse].set_xlabel(mice[iMouse])
for jMouse in range(nMice):
rezXY, rezYX, arrXX, arrXY, arrYY, arrYX = pca.paired_comparison(
dataLst[iMouse], dataLst[jMouse], dropFirst=dropFirst)
rezMat[iMouse][jMouse] = rezXY # np.mean([rezXY, rezYX])
ax[iMouse][jMouse].plot(arrXX, label='XX')
ax[iMouse][jMouse].plot(arrXY, label='XY')
ax[iMouse][jMouse].legend()
plt.savefig('pca_consistency_bymouse_evals_' + fnameSuffix +
'.svg')
plt.close()
fig, ax = plt.subplots()
imshow(fig,
ax,
rezMat,
haveColorBar=True,
limits=[0, 1],
xTicks=mice,
yTicks=mice)
plt.savefig('pca_consistency_bymouse_metric_' + fnameSuffix +
'.svg')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2)
dfConsistency = pd_append_row(
dfConsistency, [datatype, intervName, avgConsistency])
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistency, *dfColumns)
sns.heatmap(data=dfPivot, ax=ax, annot=True, vmin=0, vmax=1, cmap='jet')
fig.savefig('consistency_coactivity_metric.svg')
plt.close()
def plot_corr_consistency_l1_phase(dataDB,
nDropPCA=None,
dropChannels=None,
performance=None,
datatype=None):
mice = sorted(dataDB.mice)
phases = dataDB.get_interval_names()
nPhases = len(phases)
dfColumns = ['mousename', 'trialtype', 'consistency']
dfConsistency = pd.DataFrame(columns=dfColumns)
for iMouse, mousename in enumerate(mice):
for trialType in dataDB.get_trial_type_names():
fnameSuffix = '_'.join(
[datatype, mousename, trialType,
str(performance)])
print(fnameSuffix)
corrLst = []
for intervName in phases:
kwargs = {
'datatype': datatype,
'intervName': intervName,
'trialType': trialType
}
if performance is not None:
kwargs['performance'] = performance
dataRSPLst = dataDB.get_neuro_data({'mousename': mousename},
zscoreDim='rs',
**kwargs)
dataRSP = np.concatenate(dataRSPLst, axis=0)
dataRP = np.mean(dataRSP, axis=1)
# dataRP = zscore(dataRP, axis=0)
if dropChannels is not None:
channelMask = np.ones(dataRP.shape[1]).astype(bool)
channelMask[dropChannels] = 0
dataRP = dataRP[:, channelMask]
if nDropPCA is not None:
dataRP = drop_PCA(dataRP, nDropPCA)
corrLst += [tril_1D(np.corrcoef(dataRP.T))]
# fig, ax = plt.subplots(nrows=nMice, ncols=nMice, figsize=(4 * nMice, 4 * nMice))
pairDict = {}
rezMat = np.zeros((nPhases, nPhases))
for idxNamei, iName in enumerate(phases):
pairDict[iName] = corrLst[idxNamei]
for idxNamej, jName in enumerate(phases):
rezMat[idxNamei][idxNamej] = np.corrcoef(
corrLst[idxNamei], corrLst[idxNamej])[0, 1]
pPlot = sns.pairplot(data=pd.DataFrame(pairDict),
vars=phases,
kind='kde')
prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str(
nDropPCA) + '/scatter/'
make_path(prefixPath)
plt.savefig(prefixPath + 'scatter_' + fnameSuffix + '.svg')
plt.close()
fig, ax = plt.subplots()
imshow(fig,
ax,
rezMat,
haveColorBar=True,
limits=[0, 1],
xTicks=phases,
yTicks=phases,
cmap='jet')
prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str(
nDropPCA) + '/metric/'
make_path(prefixPath)
plt.savefig(prefixPath + 'metric_' + fnameSuffix + '.svg')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2)
dfConsistency = pd_append_row(
dfConsistency, [mousename, trialType, avgConsistency])
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistency, *dfColumns)
sns.heatmap(data=dfPivot, ax=ax, annot=True, vmin=0, vmax=1, cmap='jet')
prefixPath = 'pics/consistency/corr/phase/dropPCA_' + str(nDropPCA) + '/'
make_path(prefixPath)
fig.savefig(prefixPath + datatype + '_' + str(performance) + '.svg')
plt.close()
def plot_corr_consistency_l1_mouse(
dataDB,
nDropPCA=None,
dropChannels=None,
exclQueryLst=None,
**kwargs): # performances=None, trialTypes=None,
assert 'intervName' in kwargs.keys(), 'Requires phases'
dps = DataParameterSweep(dataDB,
exclQueryLst,
mousename='auto',
intervName='auto',
datatype='auto',
**kwargs)
mice = sorted(dataDB.mice)
nMice = len(mice)
for paramExtraVals, dfTmp in dps.sweepDF.groupby(
dps.invert_param(['mousename', 'datatype', 'intervName'])):
plotExtraSuffix = param_vals_to_suffix(paramExtraVals)
dfColumns = ['datatype', 'phase', 'consistency']
dfConsistency = pd.DataFrame(columns=dfColumns)
for paramVals, dfMouse in dfTmp.groupby(dps.invert_param(['mousename'
])):
plotSuffix = param_vals_to_suffix(paramVals)
print(plotSuffix)
corrLst = []
for idx, row in dfMouse.iterrows():
plotSuffix = '_'.join([str(s) for s in row.values])
print(plotSuffix)
kwargsThis = pd_row_to_kwargs(row,
parseNone=True,
dropKeys=['mousename'])
dataRSPLst = dataDB.get_neuro_data(
{
'mousename': row['mousename']
}, # NOTE: zscore channels for each session to avoid session-wise effects
zscoreDim='rs',
**kwargsThis)
dataRSP = np.concatenate(dataRSPLst, axis=0)
dataRP = np.mean(dataRSP, axis=1)
# dataRP = zscore(dataRP, axis=0)
if dropChannels is not None:
channelMask = np.ones(dataRP.shape[1]).astype(bool)
channelMask[dropChannels] = 0
dataRP = dataRP[:, channelMask]
if nDropPCA is not None:
dataRP = drop_PCA(dataRP, nDropPCA)
corrLst += [tril_1D(np.corrcoef(dataRP.T))]
# fig, ax = plt.subplots(nrows=nMice, ncols=nMice, figsize=(4 * nMice, 4 * nMice))
pairDict = {}
rezMat = np.zeros((nMice, nMice))
for iMouse, mousename in enumerate(mice):
# ax[iMouse][0].set_ylabel(mice[iMouse])
# ax[-1][iMouse].set_xlabel(mice[iMouse])
pairDict[mousename] = corrLst[iMouse]
for jMouse in range(nMice):
# rezMat[iMouse][jMouse] = 1 - rmae(corrLst[iMouse], corrLst[jMouse])
rezMat[iMouse][jMouse] = np.corrcoef(
corrLst[iMouse], corrLst[jMouse])[0, 1]
# cci = offdiag_1D(corrLst[iMouse])
# ccj = offdiag_1D(corrLst[jMouse])
# ax[iMouse][jMouse].plot(cci, ccj, '.')
pPlot = sns.pairplot(data=pd.DataFrame(pairDict),
vars=mice,
kind='kde')
prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str(
nDropPCA) + '/scatter/'
make_path(prefixPath)
plt.savefig(prefixPath + 'scatter_' + plotSuffix + '.svg')
plt.close()
fig, ax = plt.subplots()
imshow(fig,
ax,
rezMat,
haveColorBar=True,
limits=[0, 1],
xTicks=mice,
yTicks=mice,
cmap='jet')
prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str(
nDropPCA) + '/metric/'
make_path(prefixPath)
plt.savefig(prefixPath + 'metric_' + plotSuffix + '.svg')
plt.close()
avgConsistency = np.round(np.mean(offdiag_1D(rezMat)), 2)
dfConsistency = pd_append_row(
dfConsistency,
[row['datatype'], row['intervName'], avgConsistency])
fig, ax = plt.subplots()
dfPivot = pd_pivot(dfConsistency, *dfColumns)
sns.heatmap(data=dfPivot,
ax=ax,
annot=True,
vmin=0,
vmax=1,
cmap='jet')
prefixPath = 'pics/consistency/corr/mouse/dropPCA_' + str(
nDropPCA) + '/'
make_path(prefixPath)
fig.savefig(prefixPath + plotExtraSuffix + '.svg')
plt.close()
def find_parse_tdt(self, pathTDT):
print('Parsing TDT:')
for mouseName, dfMouse in self.dfSession.groupby(['mousename']):
pathMouse = os.path.join(pathTDT, mouseName)
prepcommon._testdir(pathMouse)
for dayName, dfDay in dfMouse.groupby(['dateKey']):
print('--', mouseName, dayName)
pathDay = os.path.join(pathMouse, 'TDT', dayName, 'widefield_labview')
pathAllen = os.path.join(pathDay, 'ROI_Allen', 'registration_transform_1510.mat')
prepcommon._testdir(pathDay)
prepcommon._testfile(pathAllen)
self.pathT1[mouseName][dayName] = pathAllen
for idx, row in dfDay.iterrows():
sessionSuffix = row['sessionKey']
pathSession = os.path.join(pathDay, sessionSuffix)
pathMat = os.path.join(pathSession, 'Matt_files')
pathTrialInd = os.path.join(pathMat, 'trials_ind.mat')
if mouseName == 'mou_5':
matFiles = [f for f in os.listdir(pathMat) if os.path.isfile(os.path.join(pathMat, f))]
matFiles = [f for f in matFiles if '.mat' in f]
matFiles = [f for f in matFiles if dayName[:4] in f]
assert len(matFiles) == 1
pathTrialStructMat = os.path.join(pathMat, matFiles[0])
else:
pathTrialStructMat = os.path.join(pathMat, dayName + sessionSuffix + '.mat')
if mouseName == 'mou_5':
sessionKeyLabview = '_s' + sessionSuffix[-1]
elif mouseName == 'mou_6':
sessionKeyLabview = '_s' + str(self._letters_to_sessions(sessionSuffix))
else:
sessionKeyLabview = '_' + sessionSuffix
pathTrialStructLabview = os.path.join(pathSession, mouseName + sessionKeyLabview + ''.join(dayName.split('_')) + '.txt')
pathActivePassive = os.path.join(pathMat, 'trials_with_and_wo_initial_moves_OCIA_from_movie.mat')
pathMovementVectors = os.path.join(pathMat, 'move_vectors_from_movie.mat')
prepcommon._testdir(pathSession)
prepcommon._testdir(pathMat)
prepcommon._testfile(pathTrialInd)
prepcommon._testfile(pathTrialStructMat)
prepcommon._testfile(pathTrialStructLabview, critical=False)
prepcommon._testfile(pathActivePassive, critical=False)
prepcommon._testfile(pathMovementVectors, critical=False)
self.dataPaths = pd_append_row(self.dataPaths, [
mouseName, dayName, sessionSuffix, pathSession,
pathTrialInd, pathTrialStructMat, pathTrialStructLabview,
pathActivePassive, pathMovementVectors
])