def _find_read_channel_labels(self, path):
labelFileName = join(path, "ROIs_names.mat")
if not isfile(labelFileName):
raise ValueError("Can't find file", labelFileName)
self.channelLabels = loadmat(labelFileName)['ROIs_names']
def read_neuro_files(self):
if 'neuro' in self.metaDataFrames.keys():
nNeuroFiles = self.metaDataFrames['neuro'].shape[0]
self.dataNeuronal = []
progBar = IntProgress(min=0, max=nNeuroFiles, description='Read Neuro Data:')
display(progBar) # display the bar
for idx, datapath in enumerate(self.metaDataFrames['neuro']['path']):
filepath = join(datapath, 'data.mat')
data3D = loadmat(filepath, waitRetry=3)['data']
# Crop bad channels
data3D = data3D[:, :, self.targetChannels]
# Crop to important timeframe
data3D = data3D[:, self.targetTimesteps[0] : self.targetTimesteps[1]]
# Crop bad trials
goodTrialIdxs = ~np.any(np.isnan(data3D), axis=(1, 2))
data3D = data3D[goodTrialIdxs]
self.dataNeuronal += [data3D]
progBar.value += 1
else:
print("No Neuro files loaded, skipping reading part")
def _find_read_allen_map(self, path):
'''
Find the 2D mapping file of hemispheric cortical areas according to allen brain atlas
'''
labelFileName = join(path, "L_modified.mat")
if not isfile(labelFileName):
raise ValueError("Can't find file", labelFileName)
self.allenMap = loadmat(labelFileName)['L'] # 2D map of cortical regions
self.allenIndices = sorted(list(set(self.allenMap.flatten())))[2:] # Indices of regions. Drop First two
self.allenCounts = [np.sum(self.allenMap == i) for i in self.allenIndices] # Number of pixels per region
def _find_read_channel_labels(self, path):
'''
Channel labels are brain regions associated to each channel index
The channel labels need not be consistent across mice, or even within one mouse
'''
labelFileName = join(path, "ROI_names.mat")
if not isfile(labelFileName):
raise ValueError("Can't find file", labelFileName)
self.channelLabels = loadmat(labelFileName)['ROI_names']
def pooled_mark_channel_labels(dfRawH5, dfLabels):
for idx, row in dfLabels.iterrows():
print(row['mousename'])
M = loadmat(row['path'])
rowH5 = dfRawH5[dfRawH5['mousename'] == row['mousename']]
pathH5 = list(rowH5['path'])[0]
with h5py.File(pathH5, 'a') as h5file:
if 'channelLabels' not in h5file.keys():
h5file.create_dataset('channelLabels', data=M['channel_labels'].astype('S'))
def read_paw(folderpath, verbose=True):
if verbose:
print("Processing paw folder", folderpath)
filepath = os.path.join(folderpath, 'trials.mat')
rezdict = {'trialsPaw': loadmat(filepath)['trials']}
nTrialsPaw, nTimesPaw = rezdict['trialsPaw'].shape
if nTimesPaw == 64:
freqPaw = 7
elif nTimesPaw > 250:
freqPaw = 30
else:
raise ValueError("Unexpected number of paw timesteps", nTimePaw)
rezdict['tPaw'] = np.arange(0, nTimesPaw) / freqPaw
rezdict['freqPaw'] = freqPaw
return rezdict
def read_whisk(folderpath, verbose=True):
if verbose:
print("Processing whisk folder", folderpath)
#############################
# Read whisking angle
#############################
rezdict = {
'whiskAngle':
loadmat(os.path.join(folderpath, 'whiskAngle.mat'))['whiskAngle']
}
nTimesWhisk, nTrialsWhisk = rezdict['whiskAngle'].shape
if nTimesWhisk <= 900:
freqWhisk = 40
elif nTimesWhisk >= 1600:
freqWhisk = 200
else:
freqWhisk = 40
# raise ValueError("Unexpected number of whisk timesteps", nTimesWhisk)
print("Unexpected number of whisk timesteps", nTimesWhisk)
rezdict['tWhisk'] = np.arange(0, nTimesWhisk) / freqWhisk
rezdict['whiskAbsVelocity'] = np.vstack(
(np.abs(rezdict['whiskAngle'][1:] - rezdict['whiskAngle'][:-1]) *
freqWhisk, np.zeros(nTrialsWhisk)))
#############################
# Read first touch
#############################
firstTouchFilePath = os.path.join(folderpath,
os.path.basename(folderpath) + '.txt')
if not os.path.isfile(firstTouchFilePath):
print("Warning: first touch file does not exist", firstTouchFilePath)
rezdict['firstTouch'] = None
else:
with open(firstTouchFilePath) as fLog:
rezdict['firstTouch'] = np.array(
[line.split('\t')[1] for line in fLog.readlines()[1:]],
dtype=float)
return rezdict
def validate_trials_ind_from_orig_videos(pwdH5, pwd, dfSession):
trialTypes = ['Go', 'No Go', 'Miss', 'FA']
trIndMap = {
'tr_100': 'Go',
'tr_1200': 'No Go',
'tr_MISS': 'Miss',
'tr_FA': 'FA'
}
for idx, row in dfSession.iterrows():
print(row['session'])
# Count trial types in metadata
fpathH5 = os.path.join(pwdH5, row['mousename'] + '.h5')
dfMeta = pd.read_hdf(fpathH5, '/metadata/' + row['session'])
countsMeta = {
tt: (dfMeta['trialType'] == tt).sum()
for tt in trialTypes
}
# Count trial types in trials_ind
fpathTrInd = os.path.join(pwd, row['path_rel'], 'Matt_files',
'trials_ind.mat')
mTrInd = loadmat(fpathTrInd)
countsTrInd = {}
for key, mat in mTrInd.items():
L = 1 if isinstance(mat, int) else len(mat)
if key in trIndMap:
countsTrInd[trIndMap[key]] = L
else:
assert L == 0
# Flip Go and NoGo for latter mice
if (row['mousename'] == 'm18') or (row['mousename'] == 'm20'):
countsTrInd['Go'], countsTrInd['No Go'] = countsTrInd[
'No Go'], countsTrInd['Go']
# Test the number of trials of each type
for tt in trialTypes:
assert countsMeta[tt] == countsTrInd[tt]
def read_neuro_files(self):
if 'neuro' in self.metaDataFrames.keys():
nNeuroFiles = self.metaDataFrames['neuro'].shape[0]
self.dataNeuronal = []
progBar = IntProgress(min=0, max=nNeuroFiles, description='Read Neuro Data:')
display(progBar) # display the bar
for idx, row in self.metaDataFrames['neuro'].iterrows():
matFile = loadmat(row['path'], waitRetry=3)
for k, val in matFile.items():
if 'Hit_timecourse' in k:
data = numpy_transpose_byorder(val, 'psr', self.dimOrdCanon)
if 'yasir' not in row['datatype']:
data = self._allen_normalize_data(data) # Normalize data, unless already normalized by Yasir
self.dataNeuronal += [data]
elif k == 'dp':
self.metaDataFrames['neuro'].at[idx, 'performance'] = val
progBar.value += 1
else:
print("No Neuro files loaded, skipping reading part")
def load_data_impl(self):
################################
# Extract info from form
################################
dataPath = self.loadDataGUI.pathDataLineEdit.text()
haveDeepSearch = self.loadDataGUI.pathRecursiveCheckBox.isChecked()
fnameTemplate = self.loadDataGUI.parseFnameLineEdit.text()
dataInsideKey = self.loadDataGUI.fileStructureLineEdit.text()
ftype = '.mat' if self.loadDataGUI.fileTypeComboBox.currentIndex(
) == 0 else '.h5'
extraKeys = [
k.strip()
for k in self.loadDataGUI.pathFilterKeysLineEdit.text().split(',')
]
allKeys = [ftype] + extraKeys
################################
# Get data file names
################################
if haveDeepSearch:
fileswalk = getfiles_walk(dataPath, allKeys)
filepaths = [os.path.join(dir, file) for dir, file in fileswalk]
else:
files = getfiles(dataPath, allKeys)
filepaths = [os.path.join(dataPath, file) for file in files]
################################
# Parse data file names
################################
if fnameTemplate == '':
df = pd.DataFrame({'fpaths': filepaths})
else:
df = pd.DataFrame(
merge_dicts(
[path_extract_data(s, fnameTemplate) for s in filepaths]))
################################
# Read data
################################
self.data = []
self.loadDataGUI.loadDataProgressBar.setEnabled(True)
self.loadDataGUI.loadDataProgressBar.setValue(0)
self.loadDataGUI.loadDataProgressBar.setMaximum(len(filepaths))
if ftype == '.mat':
for iPath, fpath in enumerate(filepaths):
self.data += [loadmat(fpath)[dataInsideKey][:, :, :48]]
self.loadDataGUI.loadDataProgressBar.setValue(iPath + 1)
else:
raise ValueError('Not Implemented')
################################
# Update data table
################################
df['dataname'] = 'src'
df['shape'] = [str(d.shape) for d in self.data]
self.df = df
qtable_load_from_pandas(self.gui.dataTableWidget, df)
################################
# Update plot coloring
################################
self.dataKeys = set(df.columns) - {'dataname', 'shape'}
self.gui.plotColorComboBox.clear()
self.gui.plotColorComboBox.addItems(['None'] + list(self.dataKeys))
self.loadDataDialog.close()
def load_ref_img(path):
return loadmat(path)['refImg']
def read_neuro_perf(folderpath, verbose=True, withPerformance=True):
# Read MAT file from command line
if verbose:
print("Reading Yaro data from", folderpath)
fname_data = os.path.join(folderpath, "data.mat")
fname_behaviour = os.path.join(folderpath, "behaviorvar.mat")
fpath_performance = os.path.join(folderpath, "Transfer Entropy")
waitRetry = 3 # Seconds wait before trying to reload the file if it is not accessible
data = loadmat(fname_data, waitRetry=waitRetry)['data']
nTrialsData = data.shape[0]
behavior = loadmat(fname_behaviour, waitRetry=waitRetry)
# Preprocess behaviour
behKeys = ['iGO', 'iNOGO', 'iFA', 'iMISS']
behavToArray = lambda b: np.array([b], dtype=int) if type(
b) == int else b # If array has 1 index it appears as a number :(
behavior = {k: v
for k, v in behavior.items()
if k[0] != '_'} # Get rid of useless fields in behaviour
for k in behKeys:
behavior[k] = behavToArray(behavior[k]) # Convert to arrays
# Compute performance
if withPerformance:
performance = mouse_performance_single_session(nTrialsData, behavior)
if not os.path.exists(fpath_performance):
print("--Warning: No performance metrics found for",
os.path.dirname(folderpath), "; Using calculated")
else:
fname_performance = os.path.join(fpath_performance,
"performance.mat")
performanceExt = loadmat(fname_performance,
waitRetry=waitRetry)['performance']
if performanceExt != performance:
print("Calculated performance", performance,
"does not match external", performanceExt)
# Convert trials structure to a dictionary
behavior['trials'] = merge_dicts_of_lists(
[matstruct2dict(obj) for obj in behavior['trials']])
# CONSISTENCY TEST 1 - If behavioural trials are more than neuronal, crop:
for behKey in behKeys:
behArray = behavior[behKey]
behNTrialsThis = len(behArray)
if behNTrialsThis > 0:
behMaxIdxThis = np.max(
behArray) - 1 # Note Matlab indices start from 1
if behMaxIdxThis >= nTrialsData:
print("--Warning: For", behKey, "behaviour max index",
behMaxIdxThis, "exceeds nTrials", nTrialsData)
behavior[behKey] = behavior[behKey][
behavior[behKey] < nTrialsData]
print("---Cropped excessive behaviour trials from",
behNTrialsThis, "to", len(behavior[behKey]))
# CONSISTENCY TEST 2 - If neuronal trials are more than behavioural
# Note that there are other keys except the above four, so data trials may be more than those for the four keys
behIndices = [idx for key in behKeys for idx in behavior[key]]
assert len(
behIndices
) <= nTrialsData, "After cropping behavioural trials may not exceed data trials"
# CONSISTENCY TEST 3 - Test behavioural indices for duplicates
for idx in behIndices:
thisCount = behIndices.count(idx)
if thisCount != 1:
keysRep = {
key: list(behavior[key]).count(idx)
for key in behKeys if idx in behavior[key]
}
print("--WARNING: index", idx, "appears multiple times:", keysRep)
#assert behIndices.count(idx) == 1, "Found duplicates in behaviour indices"
if withPerformance:
return data, behavior, performance
else:
return data, behavior
def read_lick(folderpath, verbose=True):
if verbose:
print("Processing lick folder", folderpath)
rez = {}
################################
# Process Reaction times file
################################
rt_file = os.path.join(folderpath, "RT_264.mat")
rt = loadmat(rt_file)
rez['reaction_time'] = 3.0 + rt['reaction_time']
################################
# Process lick_traces file
################################
def lick_filter(data, bot_th, top_th):
data[np.isnan(data)] = 0
return np.logical_or(data <= bot_th, data >= top_th).astype(int)
lick_traces_file = os.path.join(folderpath, "lick_traces.mat")
lick_traces = loadmat(lick_traces_file)
nTimesLick = len(lick_traces['licks_go'])
freqLick = 100 # Hz
rez['tLicks'] = np.arange(0, nTimesLick) / freqLick
# Top threshold is wrong sometimes. Yaro said to use exact one
thBot, thTop = lick_traces['bot_thresh'], 2.64
for k in [
'licks_go', 'licks_nogo', 'licks_miss', 'licks_FA', 'licks_early'
]:
rez[k] = lick_filter(lick_traces[k], thBot, thTop)
################################
# Process trials file
################################
TIMESCALE_TRACES = 0.001 # ms
trials_file = os.path.join(folderpath,
os.path.basename(folderpath) + ".mat")
#print(trials_file)
lick_trials = loadmat(trials_file)
# NOTE: lick_trials['licks']['lick_vector'] is just a repeat from above lick_traces file
# lick_trials['licks'] = merge_dicts([matstruct2dict(obj) for obj in lick_trials['licks']])
lick_trials['trials'] = merge_dicts_of_lists(
[matstruct2dict(obj) for obj in lick_trials['trials']])
fixearly = lambda trial: np.nan if trial == 'Early' else trial
lick_trials['trials']['reward_time'] = [
fixearly(trial) for trial in lick_trials['trials']['reward_time']
]
rez['reward_time'] = np.array(lick_trials['trials']['reward_time'],
dtype=float) * TIMESCALE_TRACES
rez['puff'] = [
np.array(puff, dtype=float) * TIMESCALE_TRACES
for puff in lick_trials['trials']['puff']
]
return rez
