def validate_script(self): """ validate the current script - by checking the existence of the files needed for the analysis :return: False if cannot find files; True if all are found """ # if self.script['module'] != 'IVCurve': # print 'Script is not for IVCurve (found %s)' % self.script['module'] # return False if 'directory' in self.script.keys(): try: #print dir(self.dataManager()) self.dataManager().setBaseDir(self.script['directory']) print('Set base dir to: {:s}'.format(self.script['directory'])) except: print( 'ScriptProcessor:read_script: \n Cannot set base directory to %s\n Likely directory was not found' % self.script['directory']) return False all_found = True trailingchars = [ c for c in map(chr, range(97, 123)) ] # trailing chars used to identify different parts of a cell's data for c in self.script['Cells']: if self.script['Cells'][c]['include'] is False: continue sortedkeys = sorted( self.script['Cells'][c] ['choice'].keys()) # sort by order of recording for p in sortedkeys: pr = self.script[ 'protocol'] + '_' + p # add the underscore here if c[-1] in trailingchars: cell = c[:-1] else: cell = c fn = os.path.join(cell, pr) #print fn #print 'dm selected file: ', self.dataManager().selectedFile() if 'directory' in self.script.keys(): dm_selected_file = self.script['directory'] else: dm_selected_file = self.dataManager().selectedFile().name() DataManager.cleanup() gc.collect() fullpath = os.path.join(dm_selected_file, fn) file_ok = os.path.exists(fullpath) if file_ok: print('File found: {:s}'.format(fullpath)) else: print(' current dataManager self.dm points to file: ', dm_selected_file) print(' and file not found was: ', fullpath) all_found = False #else: # print 'file found ok: %s' % fullpath return all_found
def doCells(self, slice): """ process all of the cells from a slice :param slice: :return nothing: """ allfiles = os.listdir(slice) celltype = re.compile("(cell_)(\d{3,3})") cells = [] for thisfile in allfiles: m = celltype.match(thisfile) if m is None: continue if len(m.groups()) == 2: cells.append(thisfile) for cell in cells: self.cellstring = '%s\t' % (cell) dh = DataManager.getDirHandle(os.path.join(slice, cell), create=False) cl = self.dataModel.getCellInfo(dh) if cl is not None and 'notes' in cl.keys() and len( cl['notes']) > 0: l = self.tw['cell'].wrap(cl['notes']) for i in l: i = i.replace( '\t', ' ' ) # clean out tabs so printed formatting is not confused self.cellstring += i else: self.cellstring += ' No cell notes' self.cellstring += ' \t' self.cell_summary(dh) # if cl is not None and 'description' in cl.keys() and len(cl['description']) > 0: # l = self.twd['cell'].wrap(cl['description']) # for i in l: # self.cellstring += i # else: # self.cellstring += ' No cell description' self.cellstring += ' \t' self.doProtocols(os.path.join(slice, cell)) DataManager.cleanup() # clean up after each cell del dh gc.collect()
def doSlices(self, day): """ process all of the slices for a given day :param day: :return nothing: """ allfiles = os.listdir(day) slicetype = re.compile("(slice\_)(\d{3,3})") slices = [] for thisfile in allfiles: m = slicetype.match(thisfile) if m is None: continue if len(m.groups()) == 2: slices.append(thisfile) for slice in slices: self.slicestring = '%s \t' % (slice) dh = DataManager.getDirHandle(os.path.join(day, slice), create=False) sl = self.dataModel.getSliceInfo(dh) # if sl is not None and 'description' in sl.keys() and len(sl['description']) > 0: # l = self.twd['slice'].wrap(sl['description']) # for i in l: # self.slicestring += i # else: # self.slicestring += ' No slice description' # self.slicestring += '\t' if sl is not None and 'notes' in sl.keys() and len( sl['notes']) > 0: l = self.tw['slice'].wrap(sl['notes']) for i in l: i = i.replace( '\t', ' ' ) # clean out tabs so printed formatting is not confused self.slicestring += i else: self.slicestring += ' No slice notes' self.slicestring += ' \t' self.doCells(os.path.join(day, slice)) DataManager.cleanup() del dh gc.collect()
def doProtocols(self, cell): """ process all of the protocols for a given cell :param cell: :return nothing: """ allfiles = os.listdir(cell) #celltype = re.compile("(Cell_)(\d{3,3})") protocols = [] nonprotocols = [] anyprotocols = False images = [] # tiff stacks2p = [] images2p = [] videos = [] endmatch = re.compile( "[\_(\d{3,3})]$") # look for _lmn at end of directory name for thisfile in allfiles: if os.path.isdir(os.path.join(cell, thisfile)): protocols.append(thisfile) else: nonprotocols.append(thisfile) self.protocolstring = '' if self.InvestigateProtocols is True: self.summarystring = 'NaN \t' * 6 for np, protocol in enumerate(protocols): dh = DataManager.getDirHandle(os.path.join(cell, protocol), create=False) if np == 0: self.cell_summary(dh) if self.monitor: print 'Investigating Protocol: %s', dh.name() dirs = dh.subDirs() protocolok = True # assume that protocol is ok modes = [] nexpected = len( dirs ) # acq4 writes dirs before, so this is the expected fill ncomplete = 0 # count number actually done clampDevices = self.dataModel.getClampDeviceNames(dh) # must handle multiple data formats, even in one experiment... if clampDevices is not None: data_mode = dh.info()['devices'][clampDevices[0]][ 'mode'] # get mode from top of protocol information else: # try to set a data mode indirectly if 'devices' not in dh.info().keys(): protocolok = False # can't parse protocol device... continue devices = dh.info()['devices'].keys( ) # try to get clamp devices from another location #print dir(self.dataModel) for kc in self.dataModel.knownClampNames(): if kc in devices: clampDevices = [kc] try: data_mode = dh.info()['devices'][ clampDevices[0]]['mode'] except: data_mode = 'Unknown' # protocolok = False # print '<<cannot read protocol data mode>>' if data_mode not in modes: modes.append(data_mode) for i, directory_name in enumerate( dirs ): # dirs has the names of the runs within the protocol data_dir_handle = dh[ directory_name] # get the directory within the protocol try: data_file_handle = self.dataModel.getClampFile( data_dir_handle) # get pointer to clamp data except: data_file_handle = None if data_file_handle is not None: # no clamp file found - skip ncomplete += 1 # Check if there is no clamp file for this iteration of the protocol # Usually this indicates that the protocol was stopped early. # data_file = data_file_handle.read() try: self.holding = self.dataModel.getClampHoldingLevel( data_file_handle) except: self.holding = 0. try: self.amp_settings = self.dataModel.getWCCompSettings( data_file_handle) except: self.amp_settings = None #raise ValueError('complete = %d when failed' % ncomplete) #else: # break # do not keep looking if the file is not found DataManager.cleanup() # close all opened files # del dh gc.collect( ) # and force garbage collection of freed objects inside the loop if modes == []: modes = ['Unknown mode'] if protocolok and ncomplete == nexpected: # accumulate protocols self.protocolstring += '[{:<s}: {:s} {:d}], '.format( protocol, modes[0][0], ncomplete) anyprotocols = True # indicate that ANY protocol ran to completion else: if self.reportIncompleteProtocols: self.protocolstring += '[{:<s}, ({:s}, {:d}/{:d}, Incomplete)], '.format( protocol, modes[0][0], ncomplete, nexpected) DataManager.cleanup() del dh gc.collect() else: self.protocolstring += 'Protocols: ' anyprotocols = True prots = {} for protocol in protocols: m = endmatch.search(protocol) if m is not None: p = protocol[:-4] else: p = protocol if p not in prots.keys(): prots[p] = 1 else: prots[p] += 1 if len(prots.keys()) > 0: self.protocolstring += '[' for p in prots.keys(): self.protocolstring += '{:<s}({:<d}), '.format(p, prots[p]) self.protocolstring += ']' else: self.protocolstring = '<No protocols found>' self.protocolstring += ' \t' for thisfile in nonprotocols: # if os.path.isdir(os.path.join(cell, thisfile)): # skip protocols # continue x = self.img_re.match(thisfile) # look for image files if x is not None: images.append(thisfile) x = self.s2p_re.match(thisfile) # two photon stacks if x is not None: stacks2p.append(thisfile) x = self.i2p_re.match(thisfile) # simple two photon images if x is not None: images2p.append(thisfile) x = self.video_re.match(thisfile) # video images if x is not None: videos.append(thisfile) self.imagestring = '' if len(images) > 0: self.imagestring += 'Images: %d ' % len(images) if len(stacks2p) > 0: self.imagestring += '2pStacks: %d ' % len(stacks2p) if len(images2p) > 0: self.imagestring += '2pImages: %d ' % len(images2p) if len(videos) > 0: self.imagestring += 'Videos: %d' % len(videos) if len(images) + len(stacks2p) + len(images2p) + len(videos) == 0: self.imagestring = 'No Images or Videos' if anyprotocols: ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + self.protocolstring + self.imagestring + ' \t' else: ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + '<No complete protocols> \t' + self.imagestring + ' \t' self.outputString(ostring)
def run_script(self): """ Run a script, doing all of the requested analysis :return: """ if self.script['testfiles']: return # settext = self.scripts_form.PSPReversal_ScriptResults_text.setPlainText # apptext = self.scripts_form.PSPReversal_ScriptResults_text.appendPlainText self.textout = ('\nScript File: {:<32s}\n'.format(self.script_name)) # settext(self.textout) script_header = True # reset the table to a print new header for each cell trailingchars = [ c for c in map(chr, range(97, 123)) ] # trailing chars used to identify different parts of a cell's data self.dataManager().setBaseDir(self.script['directory']) ordered = sorted(self.script['Cells'].keys() ) # order the analysis by date/slice/cell prog1 = ProgressDialog("Script Processing..", 0, len(ordered)) ncell = len(ordered) for nc, cell in enumerate(ordered): if prog1.wasCanceled(): break presetDict = {} thiscell = self.script['Cells'][cell] #print 'processing cell: %s' % thiscell if thiscell['include'] is False: # skip this cell try: print('Skipped: %s, reason:%s' % (cell, thiscell['reason'])) except: raise ValueError( 'cell %s has no tag "reason" but "include" is False' % cell) continue sortedkeys = sorted(thiscell['choice'].keys() ) # sort by order of recording (# on protocol) prog1.setValue(nc / ncell) # prog2 = ProgressDialog("Cell Processing..%s" , 0, len(sortedkeys)): for p in sortedkeys: if thiscell['choice'][p] not in self.script[ 'datafilter']: # pick out steady-state conditions print('p: %s not in data: ' % (thiscell['choice'][p]), self.script['datafilter']) continue # print 'working on %s' % thiscell['choice'][p] pr = self.script[ 'protocol'] + '_' + p # add the underscore here if cell[-1] in trailingchars: # check last letter - if not a number clip it cell_file = cell[:-1] else: cell_file = cell fn = os.path.join(cell_file, pr) #dm_selected_file = self.dataManager().selectedFile().name() dm_selected_file = self.script['directory'] fullpath = os.path.join(dm_selected_file, fn) file_ok = os.path.exists(fullpath) if not file_ok: # get the directory handle and take it from there print('File is not ok: %s' % fullpath) continue m = thiscell['choice'][p] # get the tag for the manipulation presetDict['Choices'] = thiscell['choice'][p] if 'genotype' in thiscell.keys(): presetDict['Genotype'] = thiscell['genotype'] else: presetDict['Genotype'] = 'Unknown' if 'Celltype' in thiscell.keys(): presetDict['Celltype'] = thiscell['Celltype'] else: presetDict['Celltype'] = 'Unknown' if 'spikethresh' in thiscell.keys(): presetDict['SpikeThreshold'] = thiscell['spikethresh'] if 'bridgeCorrection' in thiscell.keys(): presetDict['bridgeCorrection'] = thiscell[ 'bridgeCorrection'] else: presetDict['bridgeCorrection'] = None dh = self.dataManager().manager.dirHandle(fullpath) if not self.loadFile( [dh], analyze=False, bridge=presetDict['bridgeCorrection'] ): # note: must pass a list of dh; don't let analyisis run at end print('Failed to load requested file: ', fullpath) continue # skip bad sets of records... if 'datamode' in thiscell.keys(): self.clamps.data_mode = thiscell['datamode'] self.auto_updater = False self.get_script_analysisPars(self.script, thiscell) self.analysis( presets=presetDict) # call the caller's analysis routine if 'addtoDB' in self.script.keys(): if self.script[ 'addtoDB'] is True and self.dbUpdate is not None: self.dbUpdate() # call routine in parent ptxt = self.printAnalysis(printnow=False, script_header=script_header, copytoclipboard=False) self.textout += ptxt + '\n' script_header = False DataManager.cleanup() del dh gc.collect() print(self.textout) self.auto_updater = True # restore function