def validate_script(self):
"""
validate the current script - by checking the existence of the files needed for the analysis
:return: False if cannot find files; True if all are found
"""
# if self.script['module'] != 'IVCurve':
# print 'Script is not for IVCurve (found %s)' % self.script['module']
# return False
if 'directory' in self.script.keys():
try:
#print dir(self.dataManager())
self.dataManager().setBaseDir(self.script['directory'])
print('Set base dir to: {:s}'.format(self.script['directory']))
except:
print(
'ScriptProcessor:read_script: \n Cannot set base directory to %s\n Likely directory was not found'
% self.script['directory'])
return False
all_found = True
trailingchars = [
c for c in map(chr, range(97, 123))
] # trailing chars used to identify different parts of a cell's data
for c in self.script['Cells']:
if self.script['Cells'][c]['include'] is False:
continue
sortedkeys = sorted(
self.script['Cells'][c]
['choice'].keys()) # sort by order of recording
for p in sortedkeys:
pr = self.script[
'protocol'] + '_' + p # add the underscore here
if c[-1] in trailingchars:
cell = c[:-1]
else:
cell = c
fn = os.path.join(cell, pr)
#print fn
#print 'dm selected file: ', self.dataManager().selectedFile()
if 'directory' in self.script.keys():
dm_selected_file = self.script['directory']
else:
dm_selected_file = self.dataManager().selectedFile().name()
DataManager.cleanup()
gc.collect()
fullpath = os.path.join(dm_selected_file, fn)
file_ok = os.path.exists(fullpath)
if file_ok:
print('File found: {:s}'.format(fullpath))
else:
print(' current dataManager self.dm points to file: ',
dm_selected_file)
print(' and file not found was: ', fullpath)
all_found = False
#else:
# print 'file found ok: %s' % fullpath
return all_found
def doCells(self, slice):
"""
process all of the cells from a slice
:param slice:
:return nothing:
"""
allfiles = os.listdir(slice)
celltype = re.compile("(cell_)(\d{3,3})")
cells = []
for thisfile in allfiles:
m = celltype.match(thisfile)
if m is None:
continue
if len(m.groups()) == 2:
cells.append(thisfile)
for cell in cells:
self.cellstring = '%s\t' % (cell)
dh = DataManager.getDirHandle(os.path.join(slice, cell),
create=False)
cl = self.dataModel.getCellInfo(dh)
if cl is not None and 'notes' in cl.keys() and len(
cl['notes']) > 0:
l = self.tw['cell'].wrap(cl['notes'])
for i in l:
i = i.replace(
'\t', ' '
) # clean out tabs so printed formatting is not confused
self.cellstring += i
else:
self.cellstring += ' No cell notes'
self.cellstring += ' \t'
self.cell_summary(dh)
# if cl is not None and 'description' in cl.keys() and len(cl['description']) > 0:
# l = self.twd['cell'].wrap(cl['description'])
# for i in l:
# self.cellstring += i
# else:
# self.cellstring += ' No cell description'
self.cellstring += ' \t'
self.doProtocols(os.path.join(slice, cell))
DataManager.cleanup() # clean up after each cell
del dh
gc.collect()
def doSlices(self, day):
"""
process all of the slices for a given day
:param day:
:return nothing:
"""
allfiles = os.listdir(day)
slicetype = re.compile("(slice\_)(\d{3,3})")
slices = []
for thisfile in allfiles:
m = slicetype.match(thisfile)
if m is None:
continue
if len(m.groups()) == 2:
slices.append(thisfile)
for slice in slices:
self.slicestring = '%s \t' % (slice)
dh = DataManager.getDirHandle(os.path.join(day, slice),
create=False)
sl = self.dataModel.getSliceInfo(dh)
# if sl is not None and 'description' in sl.keys() and len(sl['description']) > 0:
# l = self.twd['slice'].wrap(sl['description'])
# for i in l:
# self.slicestring += i
# else:
# self.slicestring += ' No slice description'
# self.slicestring += '\t'
if sl is not None and 'notes' in sl.keys() and len(
sl['notes']) > 0:
l = self.tw['slice'].wrap(sl['notes'])
for i in l:
i = i.replace(
'\t', ' '
) # clean out tabs so printed formatting is not confused
self.slicestring += i
else:
self.slicestring += ' No slice notes'
self.slicestring += ' \t'
self.doCells(os.path.join(day, slice))
DataManager.cleanup()
del dh
gc.collect()
def doProtocols(self, cell):
"""
process all of the protocols for a given cell
:param cell:
:return nothing:
"""
allfiles = os.listdir(cell)
#celltype = re.compile("(Cell_)(\d{3,3})")
protocols = []
nonprotocols = []
anyprotocols = False
images = [] # tiff
stacks2p = []
images2p = []
videos = []
endmatch = re.compile(
"[\_(\d{3,3})]$") # look for _lmn at end of directory name
for thisfile in allfiles:
if os.path.isdir(os.path.join(cell, thisfile)):
protocols.append(thisfile)
else:
nonprotocols.append(thisfile)
self.protocolstring = ''
if self.InvestigateProtocols is True:
self.summarystring = 'NaN \t' * 6
for np, protocol in enumerate(protocols):
dh = DataManager.getDirHandle(os.path.join(cell, protocol),
create=False)
if np == 0:
self.cell_summary(dh)
if self.monitor:
print 'Investigating Protocol: %s', dh.name()
dirs = dh.subDirs()
protocolok = True # assume that protocol is ok
modes = []
nexpected = len(
dirs
) # acq4 writes dirs before, so this is the expected fill
ncomplete = 0 # count number actually done
clampDevices = self.dataModel.getClampDeviceNames(dh)
# must handle multiple data formats, even in one experiment...
if clampDevices is not None:
data_mode = dh.info()['devices'][clampDevices[0]][
'mode'] # get mode from top of protocol information
else: # try to set a data mode indirectly
if 'devices' not in dh.info().keys():
protocolok = False # can't parse protocol device...
continue
devices = dh.info()['devices'].keys(
) # try to get clamp devices from another location
#print dir(self.dataModel)
for kc in self.dataModel.knownClampNames():
if kc in devices:
clampDevices = [kc]
try:
data_mode = dh.info()['devices'][
clampDevices[0]]['mode']
except:
data_mode = 'Unknown'
# protocolok = False
# print '<<cannot read protocol data mode>>'
if data_mode not in modes:
modes.append(data_mode)
for i, directory_name in enumerate(
dirs
): # dirs has the names of the runs within the protocol
data_dir_handle = dh[
directory_name] # get the directory within the protocol
try:
data_file_handle = self.dataModel.getClampFile(
data_dir_handle) # get pointer to clamp data
except:
data_file_handle = None
if data_file_handle is not None: # no clamp file found - skip
ncomplete += 1
# Check if there is no clamp file for this iteration of the protocol
# Usually this indicates that the protocol was stopped early.
# data_file = data_file_handle.read()
try:
self.holding = self.dataModel.getClampHoldingLevel(
data_file_handle)
except:
self.holding = 0.
try:
self.amp_settings = self.dataModel.getWCCompSettings(
data_file_handle)
except:
self.amp_settings = None
#raise ValueError('complete = %d when failed' % ncomplete)
#else:
# break # do not keep looking if the file is not found
DataManager.cleanup() # close all opened files
# del dh
gc.collect(
) # and force garbage collection of freed objects inside the loop
if modes == []:
modes = ['Unknown mode']
if protocolok and ncomplete == nexpected: # accumulate protocols
self.protocolstring += '[{:<s}: {:s} {:d}], '.format(
protocol, modes[0][0], ncomplete)
anyprotocols = True # indicate that ANY protocol ran to completion
else:
if self.reportIncompleteProtocols:
self.protocolstring += '[{:<s}, ({:s}, {:d}/{:d}, Incomplete)], '.format(
protocol, modes[0][0], ncomplete, nexpected)
DataManager.cleanup()
del dh
gc.collect()
else:
self.protocolstring += 'Protocols: '
anyprotocols = True
prots = {}
for protocol in protocols:
m = endmatch.search(protocol)
if m is not None:
p = protocol[:-4]
else:
p = protocol
if p not in prots.keys():
prots[p] = 1
else:
prots[p] += 1
if len(prots.keys()) > 0:
self.protocolstring += '['
for p in prots.keys():
self.protocolstring += '{:<s}({:<d}), '.format(p, prots[p])
self.protocolstring += ']'
else:
self.protocolstring = '<No protocols found>'
self.protocolstring += ' \t'
for thisfile in nonprotocols:
# if os.path.isdir(os.path.join(cell, thisfile)): # skip protocols
# continue
x = self.img_re.match(thisfile) # look for image files
if x is not None:
images.append(thisfile)
x = self.s2p_re.match(thisfile) # two photon stacks
if x is not None:
stacks2p.append(thisfile)
x = self.i2p_re.match(thisfile) # simple two photon images
if x is not None:
images2p.append(thisfile)
x = self.video_re.match(thisfile) # video images
if x is not None:
videos.append(thisfile)
self.imagestring = ''
if len(images) > 0:
self.imagestring += 'Images: %d ' % len(images)
if len(stacks2p) > 0:
self.imagestring += '2pStacks: %d ' % len(stacks2p)
if len(images2p) > 0:
self.imagestring += '2pImages: %d ' % len(images2p)
if len(videos) > 0:
self.imagestring += 'Videos: %d' % len(videos)
if len(images) + len(stacks2p) + len(images2p) + len(videos) == 0:
self.imagestring = 'No Images or Videos'
if anyprotocols:
ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + self.protocolstring + self.imagestring + ' \t'
else:
ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + '<No complete protocols> \t' + self.imagestring + ' \t'
self.outputString(ostring)
def run_script(self):
"""
Run a script, doing all of the requested analysis
:return:
"""
if self.script['testfiles']:
return
# settext = self.scripts_form.PSPReversal_ScriptResults_text.setPlainText
# apptext = self.scripts_form.PSPReversal_ScriptResults_text.appendPlainText
self.textout = ('\nScript File: {:<32s}\n'.format(self.script_name))
# settext(self.textout)
script_header = True # reset the table to a print new header for each cell
trailingchars = [
c for c in map(chr, range(97, 123))
] # trailing chars used to identify different parts of a cell's data
self.dataManager().setBaseDir(self.script['directory'])
ordered = sorted(self.script['Cells'].keys()
) # order the analysis by date/slice/cell
prog1 = ProgressDialog("Script Processing..", 0, len(ordered))
ncell = len(ordered)
for nc, cell in enumerate(ordered):
if prog1.wasCanceled():
break
presetDict = {}
thiscell = self.script['Cells'][cell]
#print 'processing cell: %s' % thiscell
if thiscell['include'] is False: # skip this cell
try:
print('Skipped: %s, reason:%s' %
(cell, thiscell['reason']))
except:
raise ValueError(
'cell %s has no tag "reason" but "include" is False' %
cell)
continue
sortedkeys = sorted(thiscell['choice'].keys()
) # sort by order of recording (# on protocol)
prog1.setValue(nc / ncell)
# prog2 = ProgressDialog("Cell Processing..%s" , 0, len(sortedkeys)):
for p in sortedkeys:
if thiscell['choice'][p] not in self.script[
'datafilter']: # pick out steady-state conditions
print('p: %s not in data: ' % (thiscell['choice'][p]),
self.script['datafilter'])
continue
# print 'working on %s' % thiscell['choice'][p]
pr = self.script[
'protocol'] + '_' + p # add the underscore here
if cell[-1] in trailingchars: # check last letter - if not a number clip it
cell_file = cell[:-1]
else:
cell_file = cell
fn = os.path.join(cell_file, pr)
#dm_selected_file = self.dataManager().selectedFile().name()
dm_selected_file = self.script['directory']
fullpath = os.path.join(dm_selected_file, fn)
file_ok = os.path.exists(fullpath)
if not file_ok: # get the directory handle and take it from there
print('File is not ok: %s' % fullpath)
continue
m = thiscell['choice'][p] # get the tag for the manipulation
presetDict['Choices'] = thiscell['choice'][p]
if 'genotype' in thiscell.keys():
presetDict['Genotype'] = thiscell['genotype']
else:
presetDict['Genotype'] = 'Unknown'
if 'Celltype' in thiscell.keys():
presetDict['Celltype'] = thiscell['Celltype']
else:
presetDict['Celltype'] = 'Unknown'
if 'spikethresh' in thiscell.keys():
presetDict['SpikeThreshold'] = thiscell['spikethresh']
if 'bridgeCorrection' in thiscell.keys():
presetDict['bridgeCorrection'] = thiscell[
'bridgeCorrection']
else:
presetDict['bridgeCorrection'] = None
dh = self.dataManager().manager.dirHandle(fullpath)
if not self.loadFile(
[dh], analyze=False, bridge=presetDict['bridgeCorrection']
): # note: must pass a list of dh; don't let analyisis run at end
print('Failed to load requested file: ', fullpath)
continue # skip bad sets of records...
if 'datamode' in thiscell.keys():
self.clamps.data_mode = thiscell['datamode']
self.auto_updater = False
self.get_script_analysisPars(self.script, thiscell)
self.analysis(
presets=presetDict) # call the caller's analysis routine
if 'addtoDB' in self.script.keys():
if self.script[
'addtoDB'] is True and self.dbUpdate is not None:
self.dbUpdate() # call routine in parent
ptxt = self.printAnalysis(printnow=False,
script_header=script_header,
copytoclipboard=False)
self.textout += ptxt + '\n'
script_header = False
DataManager.cleanup()
del dh
gc.collect()
print(self.textout)
self.auto_updater = True # restore function
