def _find_read_channel_labels(self, path): labelFileName = join(path, "ROIs_names.mat") if not isfile(labelFileName): raise ValueError("Can't find file", labelFileName) self.channelLabels = loadmat(labelFileName)['ROIs_names']
def read_neuro_files(self): if 'neuro' in self.metaDataFrames.keys(): nNeuroFiles = self.metaDataFrames['neuro'].shape[0] self.dataNeuronal = [] progBar = IntProgress(min=0, max=nNeuroFiles, description='Read Neuro Data:') display(progBar) # display the bar for idx, datapath in enumerate(self.metaDataFrames['neuro']['path']): filepath = join(datapath, 'data.mat') data3D = loadmat(filepath, waitRetry=3)['data'] # Crop bad channels data3D = data3D[:, :, self.targetChannels] # Crop to important timeframe data3D = data3D[:, self.targetTimesteps[0] : self.targetTimesteps[1]] # Crop bad trials goodTrialIdxs = ~np.any(np.isnan(data3D), axis=(1, 2)) data3D = data3D[goodTrialIdxs] self.dataNeuronal += [data3D] progBar.value += 1 else: print("No Neuro files loaded, skipping reading part")
def _find_read_allen_map(self, path): ''' Find the 2D mapping file of hemispheric cortical areas according to allen brain atlas ''' labelFileName = join(path, "L_modified.mat") if not isfile(labelFileName): raise ValueError("Can't find file", labelFileName) self.allenMap = loadmat(labelFileName)['L'] # 2D map of cortical regions self.allenIndices = sorted(list(set(self.allenMap.flatten())))[2:] # Indices of regions. Drop First two self.allenCounts = [np.sum(self.allenMap == i) for i in self.allenIndices] # Number of pixels per region
def _find_read_channel_labels(self, path): ''' Channel labels are brain regions associated to each channel index The channel labels need not be consistent across mice, or even within one mouse ''' labelFileName = join(path, "ROI_names.mat") if not isfile(labelFileName): raise ValueError("Can't find file", labelFileName) self.channelLabels = loadmat(labelFileName)['ROI_names']
def pooled_mark_channel_labels(dfRawH5, dfLabels): for idx, row in dfLabels.iterrows(): print(row['mousename']) M = loadmat(row['path']) rowH5 = dfRawH5[dfRawH5['mousename'] == row['mousename']] pathH5 = list(rowH5['path'])[0] with h5py.File(pathH5, 'a') as h5file: if 'channelLabels' not in h5file.keys(): h5file.create_dataset('channelLabels', data=M['channel_labels'].astype('S'))
def read_paw(folderpath, verbose=True): if verbose: print("Processing paw folder", folderpath) filepath = os.path.join(folderpath, 'trials.mat') rezdict = {'trialsPaw': loadmat(filepath)['trials']} nTrialsPaw, nTimesPaw = rezdict['trialsPaw'].shape if nTimesPaw == 64: freqPaw = 7 elif nTimesPaw > 250: freqPaw = 30 else: raise ValueError("Unexpected number of paw timesteps", nTimePaw) rezdict['tPaw'] = np.arange(0, nTimesPaw) / freqPaw rezdict['freqPaw'] = freqPaw return rezdict
def read_whisk(folderpath, verbose=True): if verbose: print("Processing whisk folder", folderpath) ############################# # Read whisking angle ############################# rezdict = { 'whiskAngle': loadmat(os.path.join(folderpath, 'whiskAngle.mat'))['whiskAngle'] } nTimesWhisk, nTrialsWhisk = rezdict['whiskAngle'].shape if nTimesWhisk <= 900: freqWhisk = 40 elif nTimesWhisk >= 1600: freqWhisk = 200 else: freqWhisk = 40 # raise ValueError("Unexpected number of whisk timesteps", nTimesWhisk) print("Unexpected number of whisk timesteps", nTimesWhisk) rezdict['tWhisk'] = np.arange(0, nTimesWhisk) / freqWhisk rezdict['whiskAbsVelocity'] = np.vstack( (np.abs(rezdict['whiskAngle'][1:] - rezdict['whiskAngle'][:-1]) * freqWhisk, np.zeros(nTrialsWhisk))) ############################# # Read first touch ############################# firstTouchFilePath = os.path.join(folderpath, os.path.basename(folderpath) + '.txt') if not os.path.isfile(firstTouchFilePath): print("Warning: first touch file does not exist", firstTouchFilePath) rezdict['firstTouch'] = None else: with open(firstTouchFilePath) as fLog: rezdict['firstTouch'] = np.array( [line.split('\t')[1] for line in fLog.readlines()[1:]], dtype=float) return rezdict
def validate_trials_ind_from_orig_videos(pwdH5, pwd, dfSession): trialTypes = ['Go', 'No Go', 'Miss', 'FA'] trIndMap = { 'tr_100': 'Go', 'tr_1200': 'No Go', 'tr_MISS': 'Miss', 'tr_FA': 'FA' } for idx, row in dfSession.iterrows(): print(row['session']) # Count trial types in metadata fpathH5 = os.path.join(pwdH5, row['mousename'] + '.h5') dfMeta = pd.read_hdf(fpathH5, '/metadata/' + row['session']) countsMeta = { tt: (dfMeta['trialType'] == tt).sum() for tt in trialTypes } # Count trial types in trials_ind fpathTrInd = os.path.join(pwd, row['path_rel'], 'Matt_files', 'trials_ind.mat') mTrInd = loadmat(fpathTrInd) countsTrInd = {} for key, mat in mTrInd.items(): L = 1 if isinstance(mat, int) else len(mat) if key in trIndMap: countsTrInd[trIndMap[key]] = L else: assert L == 0 # Flip Go and NoGo for latter mice if (row['mousename'] == 'm18') or (row['mousename'] == 'm20'): countsTrInd['Go'], countsTrInd['No Go'] = countsTrInd[ 'No Go'], countsTrInd['Go'] # Test the number of trials of each type for tt in trialTypes: assert countsMeta[tt] == countsTrInd[tt]
def read_neuro_files(self): if 'neuro' in self.metaDataFrames.keys(): nNeuroFiles = self.metaDataFrames['neuro'].shape[0] self.dataNeuronal = [] progBar = IntProgress(min=0, max=nNeuroFiles, description='Read Neuro Data:') display(progBar) # display the bar for idx, row in self.metaDataFrames['neuro'].iterrows(): matFile = loadmat(row['path'], waitRetry=3) for k, val in matFile.items(): if 'Hit_timecourse' in k: data = numpy_transpose_byorder(val, 'psr', self.dimOrdCanon) if 'yasir' not in row['datatype']: data = self._allen_normalize_data(data) # Normalize data, unless already normalized by Yasir self.dataNeuronal += [data] elif k == 'dp': self.metaDataFrames['neuro'].at[idx, 'performance'] = val progBar.value += 1 else: print("No Neuro files loaded, skipping reading part")
def load_data_impl(self): ################################ # Extract info from form ################################ dataPath = self.loadDataGUI.pathDataLineEdit.text() haveDeepSearch = self.loadDataGUI.pathRecursiveCheckBox.isChecked() fnameTemplate = self.loadDataGUI.parseFnameLineEdit.text() dataInsideKey = self.loadDataGUI.fileStructureLineEdit.text() ftype = '.mat' if self.loadDataGUI.fileTypeComboBox.currentIndex( ) == 0 else '.h5' extraKeys = [ k.strip() for k in self.loadDataGUI.pathFilterKeysLineEdit.text().split(',') ] allKeys = [ftype] + extraKeys ################################ # Get data file names ################################ if haveDeepSearch: fileswalk = getfiles_walk(dataPath, allKeys) filepaths = [os.path.join(dir, file) for dir, file in fileswalk] else: files = getfiles(dataPath, allKeys) filepaths = [os.path.join(dataPath, file) for file in files] ################################ # Parse data file names ################################ if fnameTemplate == '': df = pd.DataFrame({'fpaths': filepaths}) else: df = pd.DataFrame( merge_dicts( [path_extract_data(s, fnameTemplate) for s in filepaths])) ################################ # Read data ################################ self.data = [] self.loadDataGUI.loadDataProgressBar.setEnabled(True) self.loadDataGUI.loadDataProgressBar.setValue(0) self.loadDataGUI.loadDataProgressBar.setMaximum(len(filepaths)) if ftype == '.mat': for iPath, fpath in enumerate(filepaths): self.data += [loadmat(fpath)[dataInsideKey][:, :, :48]] self.loadDataGUI.loadDataProgressBar.setValue(iPath + 1) else: raise ValueError('Not Implemented') ################################ # Update data table ################################ df['dataname'] = 'src' df['shape'] = [str(d.shape) for d in self.data] self.df = df qtable_load_from_pandas(self.gui.dataTableWidget, df) ################################ # Update plot coloring ################################ self.dataKeys = set(df.columns) - {'dataname', 'shape'} self.gui.plotColorComboBox.clear() self.gui.plotColorComboBox.addItems(['None'] + list(self.dataKeys)) self.loadDataDialog.close()
def load_ref_img(path): return loadmat(path)['refImg']
def read_neuro_perf(folderpath, verbose=True, withPerformance=True): # Read MAT file from command line if verbose: print("Reading Yaro data from", folderpath) fname_data = os.path.join(folderpath, "data.mat") fname_behaviour = os.path.join(folderpath, "behaviorvar.mat") fpath_performance = os.path.join(folderpath, "Transfer Entropy") waitRetry = 3 # Seconds wait before trying to reload the file if it is not accessible data = loadmat(fname_data, waitRetry=waitRetry)['data'] nTrialsData = data.shape[0] behavior = loadmat(fname_behaviour, waitRetry=waitRetry) # Preprocess behaviour behKeys = ['iGO', 'iNOGO', 'iFA', 'iMISS'] behavToArray = lambda b: np.array([b], dtype=int) if type( b) == int else b # If array has 1 index it appears as a number :( behavior = {k: v for k, v in behavior.items() if k[0] != '_'} # Get rid of useless fields in behaviour for k in behKeys: behavior[k] = behavToArray(behavior[k]) # Convert to arrays # Compute performance if withPerformance: performance = mouse_performance_single_session(nTrialsData, behavior) if not os.path.exists(fpath_performance): print("--Warning: No performance metrics found for", os.path.dirname(folderpath), "; Using calculated") else: fname_performance = os.path.join(fpath_performance, "performance.mat") performanceExt = loadmat(fname_performance, waitRetry=waitRetry)['performance'] if performanceExt != performance: print("Calculated performance", performance, "does not match external", performanceExt) # Convert trials structure to a dictionary behavior['trials'] = merge_dicts_of_lists( [matstruct2dict(obj) for obj in behavior['trials']]) # CONSISTENCY TEST 1 - If behavioural trials are more than neuronal, crop: for behKey in behKeys: behArray = behavior[behKey] behNTrialsThis = len(behArray) if behNTrialsThis > 0: behMaxIdxThis = np.max( behArray) - 1 # Note Matlab indices start from 1 if behMaxIdxThis >= nTrialsData: print("--Warning: For", behKey, "behaviour max index", behMaxIdxThis, "exceeds nTrials", nTrialsData) behavior[behKey] = behavior[behKey][ behavior[behKey] < nTrialsData] print("---Cropped excessive behaviour trials from", behNTrialsThis, "to", len(behavior[behKey])) # CONSISTENCY TEST 2 - If neuronal trials are more than behavioural # Note that there are other keys except the above four, so data trials may be more than those for the four keys behIndices = [idx for key in behKeys for idx in behavior[key]] assert len( behIndices ) <= nTrialsData, "After cropping behavioural trials may not exceed data trials" # CONSISTENCY TEST 3 - Test behavioural indices for duplicates for idx in behIndices: thisCount = behIndices.count(idx) if thisCount != 1: keysRep = { key: list(behavior[key]).count(idx) for key in behKeys if idx in behavior[key] } print("--WARNING: index", idx, "appears multiple times:", keysRep) #assert behIndices.count(idx) == 1, "Found duplicates in behaviour indices" if withPerformance: return data, behavior, performance else: return data, behavior
def read_lick(folderpath, verbose=True): if verbose: print("Processing lick folder", folderpath) rez = {} ################################ # Process Reaction times file ################################ rt_file = os.path.join(folderpath, "RT_264.mat") rt = loadmat(rt_file) rez['reaction_time'] = 3.0 + rt['reaction_time'] ################################ # Process lick_traces file ################################ def lick_filter(data, bot_th, top_th): data[np.isnan(data)] = 0 return np.logical_or(data <= bot_th, data >= top_th).astype(int) lick_traces_file = os.path.join(folderpath, "lick_traces.mat") lick_traces = loadmat(lick_traces_file) nTimesLick = len(lick_traces['licks_go']) freqLick = 100 # Hz rez['tLicks'] = np.arange(0, nTimesLick) / freqLick # Top threshold is wrong sometimes. Yaro said to use exact one thBot, thTop = lick_traces['bot_thresh'], 2.64 for k in [ 'licks_go', 'licks_nogo', 'licks_miss', 'licks_FA', 'licks_early' ]: rez[k] = lick_filter(lick_traces[k], thBot, thTop) ################################ # Process trials file ################################ TIMESCALE_TRACES = 0.001 # ms trials_file = os.path.join(folderpath, os.path.basename(folderpath) + ".mat") #print(trials_file) lick_trials = loadmat(trials_file) # NOTE: lick_trials['licks']['lick_vector'] is just a repeat from above lick_traces file # lick_trials['licks'] = merge_dicts([matstruct2dict(obj) for obj in lick_trials['licks']]) lick_trials['trials'] = merge_dicts_of_lists( [matstruct2dict(obj) for obj in lick_trials['trials']]) fixearly = lambda trial: np.nan if trial == 'Early' else trial lick_trials['trials']['reward_time'] = [ fixearly(trial) for trial in lick_trials['trials']['reward_time'] ] rez['reward_time'] = np.array(lick_trials['trials']['reward_time'], dtype=float) * TIMESCALE_TRACES rez['puff'] = [ np.array(puff, dtype=float) * TIMESCALE_TRACES for puff in lick_trials['trials']['puff'] ] return rez