def test_datamanager():
rh = dm.getDirHandle(root)
# check handles are cached
rh2 = dm.getDirHandle(root)
assert rh is rh2
# test meta info is stored and reloaded correctly
rh.setInfo({'test_int': 1, 'test_str': 'xxx'})
assert rh.info()['test_int'] == 1
assert rh.info()['test_str'] == 'xxx'
# Create a subdir with meta info
d1 = rh.mkdir('subdir', info={'a': 'b'})
assert d1.info()['a'] == 'b'
assert d1.shortName() == 'subdir'
assert d1.name() == os.path.join(rh.name(), 'subdir')
# Create a DirTreeWidget; check that the contents are correct.
dw = DirTreeWidget(baseDirHandle=rh)
assert dw.topLevelItemCount() == 1
item = dw.topLevelItem(0)
assert item.text(0) == d1.shortName()
assert item.handle is d1
assert item.childCount() == 0
# Create a subdir and update the tree widget
d2 = d1.mkdir('subdir2')
dw.rebuildChildren(item)
assert item.childCount() == 1
# test _getTree
d3 = rh.mkdir('subdir3')
assert d3.name() not in dm.dm._getTree(d1.name())
assert d2.name() in dm.dm._getTree(d1.name())
#
# root
# + subdir
# | + subdir2
# + subdir3
#
assert d1.name(relativeTo=rh) == 'subdir'
assert d2.name(relativeTo=rh) == os.path.join('subdir', 'subdir2')
assert d2.name(relativeTo=d1) == 'subdir2'
assert d2.name(relativeTo=d2) == ''
assert d1.name(relativeTo=d2) == '..'
assert rh.name(relativeTo=d2) == os.path.join('..', '..')
assert d3.name(relativeTo=d2) == os.path.join('..', '..', 'subdir3')
assert d2.name(relativeTo=d3) == os.path.join('..', 'subdir', 'subdir2')
# rename subdir from tree widget
item.setText(0, 'subdir_renamed')
assert d1.shortName() == 'subdir_renamed'
# delete subdir
d1.delete()
dw.rebuildTree()
assert dw.topLevelItemCount() == 1
def restoreState(self, state, rootPath=None):
if state.get('contents', None) != 'MosaicEditor_save':
raise TypeError("This does not appear to be MosaicEditor save data.")
if state['version'][0] > self._saveVersion[0]:
raise TypeError("Save data has version %d.%d, but this MosaicEditor only supports up to version %d.x." % (state['version'][0], state['version'][1], self._saveVersion[0]))
if not self.clear():
return
root = state['rootPath']
if root == '':
# data was stored with no root path; filenames should be absolute
root = None
else:
# data was stored with no root path; filenames should be relative to the loaded file
root = DataManager.getHandle(rootPath)
for itemState in state['items']:
fname = itemState['filename']
if root is None:
fh = DataManager.getHandle(fh)
else:
fh = root[fname]
item = self.addFile(fh, name=itemState['name'], inheritTransform=False)
item.restoreState(itemState)
self.canvas.view.setState(state['view'])
def getSiteNWBs(basepath):
""" Get all experiment NWBs in the slice*/site* folders """
matches = []
dirHandleBase = adm.getHandle(basepath)
dirHandleBase.checkIndex()
# start in basepath
for k in dirHandleBase.ls():
if not dirHandleBase.isManaged(k):
continue
slicePath = os.path.join(basepath, k)
if os.path.isfile(slicePath):
continue
# base/slice
dirHandleSlice = adm.getHandle(slicePath)
dirHandleSlice.checkIndex()
for k in dirHandleSlice.ls():
if not dirHandleSlice.isManaged(k):
continue
sitePath = os.path.join(slicePath, k)
if os.path.isfile(sitePath):
continue
# base/slice/site
all_nwbs = glob.glob(os.path.join(sitePath, '*.nwb'))
all_pxps = glob.glob(os.path.join(sitePath, '*.pxp'))
if len(all_nwbs) != 1:
print "The site folder \"%s\" will be ignored as it holds more than one NWB file." % sitePath
print all_nwbs
continue
if len(all_pxps) != 1:
print "The site folder \"%s\" will be ignored as it holds more than one PXP file." % sitePath
print all_pxps
continue
siteNWB = all_nwbs[0]
sitePXP = all_pxps[0]
if not os.path.isfile(siteNWB):
print "The site folder \"%s\" will be ignored as it is missing the mandatory NWB file." % sitePath
continue
if not os.path.isfile(sitePXP):
print "The site folder \"%s\" will be ignored as it is missing the mandatory PXP file." % sitePath
continue
matches.append(os.path.abspath(siteNWB))
return matches
def validate_script(self):
"""
validate the current script - by checking the existence of the files needed for the analysis
:return: False if cannot find files; True if all are found
"""
# if self.script['module'] != 'IVCurve':
# print 'Script is not for IVCurve (found %s)' % self.script['module']
# return False
if 'directory' in self.script.keys():
try:
#print dir(self.dataManager())
self.dataManager().setBaseDir(self.script['directory'])
print('Set base dir to: {:s}'.format(self.script['directory']))
except:
print(
'ScriptProcessor:read_script: \n Cannot set base directory to %s\n Likely directory was not found'
% self.script['directory'])
return False
all_found = True
trailingchars = [
c for c in map(chr, range(97, 123))
] # trailing chars used to identify different parts of a cell's data
for c in self.script['Cells']:
if self.script['Cells'][c]['include'] is False:
continue
sortedkeys = sorted(
self.script['Cells'][c]
['choice'].keys()) # sort by order of recording
for p in sortedkeys:
pr = self.script[
'protocol'] + '_' + p # add the underscore here
if c[-1] in trailingchars:
cell = c[:-1]
else:
cell = c
fn = os.path.join(cell, pr)
#print fn
#print 'dm selected file: ', self.dataManager().selectedFile()
if 'directory' in self.script.keys():
dm_selected_file = self.script['directory']
else:
dm_selected_file = self.dataManager().selectedFile().name()
DataManager.cleanup()
gc.collect()
fullpath = os.path.join(dm_selected_file, fn)
file_ok = os.path.exists(fullpath)
if file_ok:
print('File found: {:s}'.format(fullpath))
else:
print(' current dataManager self.dm points to file: ',
dm_selected_file)
print(' and file not found was: ', fullpath)
all_found = False
#else:
# print 'file found ok: %s' % fullpath
return all_found
def restoreState(self, state, rootPath=None):
if state.get('contents', None) != 'MosaicEditor_save':
raise TypeError(
"This does not appear to be MosaicEditor save data.")
if state['version'][0] > self._saveVersion[0]:
raise TypeError(
"Save data has version %d.%d, but this MosaicEditor only supports up to version %d.x."
% (state['version'][0], state['version'][1],
self._saveVersion[0]))
if not self.clear():
return
root = state['rootPath']
if root == '':
# data was stored with no root path; filenames should be absolute
root = None
else:
# data was stored with no root path; filenames should be relative to the loaded file
root = DataManager.getHandle(rootPath)
loadfail = []
for itemState in state['items']:
fname = itemState.get('filename')
if fname is None:
# create item from scratch and restore state
itemtype = itemState.get('type')
if itemtype not in items.itemTypes():
# warn the user later on that we could not load this item
loadfail.append((itemState.get('name'),
'Unknown item type "%s"' % itemtype))
continue
item = self.addItem(type=itemtype, name=itemState['name'])
else:
# normalize file name for this OS
revsep = {'/': '\\', '\\': '/'}[os.path.sep]
fname = fname.replace(revsep, os.path.sep)
# create item by loading file and restore state
if root is None:
fh = DataManager.getHandle(fh)
else:
fh = root[fname]
item = self.addFile(fh,
name=itemState['name'],
inheritTransform=False)
item.restoreState(itemState)
self.canvas.view.setState(state['view'])
if len(loadfail) > 0:
msg = "\n".join(["%s: %s" % m for m in loadfail])
raise Exception("Failed to load some items:\n%s" % msg)
def getSiteNWBs(basepath):
""" Get all experiment NWBs in the slice*/site* folders """
matches = []
dirHandleBase = adm.getHandle(basepath)
dirHandleBase.checkIndex()
# start in basepath
for k in dirHandleBase.ls():
if not dirHandleBase.isManaged(k):
continue
slicePath = os.path.join(basepath, k)
if os.path.isfile(slicePath):
continue
# base/slice
dirHandleSlice = adm.getHandle(slicePath)
dirHandleSlice.checkIndex()
for k in dirHandleSlice.ls():
if not dirHandleSlice.isManaged(k):
continue
sitePath = os.path.join(slicePath, k)
if os.path.isfile(sitePath):
continue
# base/slice/site
all_nwbs = glob.glob(os.path.join(sitePath, '*.nwb'))
if len(all_nwbs) != 1:
print "The site folder \"%s\" will be ignored as it holds not exactly one NWB file." % sitePath
print all_nwbs
continue
siteNWB = all_nwbs[0]
if not os.path.isfile(siteNWB):
print "The site folder \"%s\" will be ignored as it is missing the mandatory NWB file." % sitePath
continue
matches.append(os.path.abspath(siteNWB))
return matches
def __init__(self, baseDir, sortMode='alpha', create=False, *args):
QtGui.QWidget.__init__(self, *args)
self.ui = Ui_Form()
self.ui.setupUi(self)
if isinstance(baseDir, basestring):
baseDir = DataManager.getDirHandle(baseDir, create=create)
self.baseDir = baseDir
self.currentFile = None
#self.fileTree = DirTreeModel(baseDir)
#self.ui.fileTree.setModel(self.fileTree)
self.ui.fileTree.setSortMode(sortMode)
self.ui.fileTree.setBaseDirHandle(baseDir)
self.deleteState = 0
self.ui.deleteBtn.focusOutEvent = self.delBtnLostFocus
self.ui.newBtn.clicked.connect(self.newClicked)
self.ui.newDirBtn.clicked.connect(self.newDirClicked)
self.ui.saveBtn.clicked.connect(self.saveClicked)
self.ui.loadBtn.clicked.connect(self.loadClicked)
self.ui.saveAsBtn.clicked.connect(self.saveAsClicked)
self.ui.deleteBtn.clicked.connect(self.deleteClicked)
self.ui.fileTree.itemDoubleClicked.connect(self.loadClicked)
def __init__(self, baseDir, sortMode='alpha', create=False, *args):
QtGui.QWidget.__init__(self, *args)
self.ui = Ui_Form()
self.ui.setupUi(self)
if isinstance(baseDir, basestring):
baseDir = DataManager.getDirHandle(baseDir, create=create)
self.baseDir = baseDir
self.currentFile = None
#self.fileTree = DirTreeModel(baseDir)
#self.ui.fileTree.setModel(self.fileTree)
self.ui.fileTree.setSortMode(sortMode)
self.ui.fileTree.setBaseDirHandle(baseDir)
self.deleteState = 0
self.ui.deleteBtn.focusOutEvent = self.delBtnLostFocus
self.ui.newBtn.clicked.connect(self.newClicked)
self.ui.newDirBtn.clicked.connect(self.newDirClicked)
self.ui.saveBtn.clicked.connect(self.saveClicked)
self.ui.loadBtn.clicked.connect(self.loadClicked)
self.ui.saveAsBtn.clicked.connect(self.saveAsClicked)
self.ui.deleteBtn.clicked.connect(self.deleteClicked)
self.ui.fileTree.itemDoubleClicked.connect(self.loadClicked)
def loadFileRequested(self, files):
self.canvas = self.getElement('Canvas')
if files is None:
return
for f in files:
if f in self.files: ## Do not allow loading the same file more than once
item = self.files[f]
item.show() # just show the file; but do not load it
continue
if f.isFile(): # add specified files
item = self.canvas.addFile(f)
elif f.isDir(): # Directories are more complicated
if self.dataModel.dirType(f) == 'Cell': # If it is a cell, just add the cell "Marker" to the plot
# note: this breaks loading all images in Cell directory (need another way to do that)
item = self.canvas.addFile(f)
else: # in all other directory types, look for MetaArray files
filesindir = glob.glob(f.name() + '/*.ma')
for fd in filesindir: # add files in the directory (ma files: e.g., images, videos)
try:
fdh = DataManager.getFileHandle(fd) # open file to get handle.
except IOError:
continue # just skip file
item = self.canvas.addFile(fdh) # add it
self.amendFile(f, item)
if len(filesindir) == 0: # add protocol sequences
item = self.canvas.addFile(f)
self.canvas.selectItem(item)
self.canvas.autoRange()
def loadFileRequested(self, files):
if files is None:
return
for f in files:
if f.shortName().endswith('.mosaic'):
self.loadStateFile(f.name())
continue
if f in self.files: ## Do not allow loading the same file more than once
item = self.files[f]
item.show() # just show the file; but do not load it
continue
if f.isFile(): # add specified files
item = self.addFile(f)
elif f.isDir(): # Directories are more complicated
if self.dataModel.dirType(
f
) == 'Cell': # If it is a cell, just add the cell "Marker" to the plot
item = self.canvas.addFile(f)
else: # in all other directory types, look for MetaArray files
filesindir = glob.glob(f.name() + '/*.ma')
for fd in filesindir: # add files in the directory (ma files: e.g., images, videos)
try:
fdh = DataManager.getFileHandle(
fd) # open file to get handle.
except IOError:
continue # just skip file
item = self.addFile(fdh)
if len(filesindir) == 0: # add protocol sequences
item = self.addFile(f)
self.canvas.autoRange()
def doCells(self, slice):
"""
process all of the cells from a slice
:param slice:
:return nothing:
"""
allfiles = os.listdir(slice)
celltype = re.compile("(cell_)(\d{3,3})")
cells = []
for thisfile in allfiles:
m = celltype.match(thisfile)
if m is None:
continue
if len(m.groups()) == 2:
cells.append(thisfile)
for cell in cells:
self.cellstring = '%s\t' % (cell)
dh = DataManager.getDirHandle(os.path.join(slice, cell),
create=False)
cl = self.dataModel.getCellInfo(dh)
if cl is not None and 'notes' in cl.keys() and len(
cl['notes']) > 0:
l = self.tw['cell'].wrap(cl['notes'])
for i in l:
i = i.replace(
'\t', ' '
) # clean out tabs so printed formatting is not confused
self.cellstring += i
else:
self.cellstring += ' No cell notes'
self.cellstring += ' \t'
self.cell_summary(dh)
# if cl is not None and 'description' in cl.keys() and len(cl['description']) > 0:
# l = self.twd['cell'].wrap(cl['description'])
# for i in l:
# self.cellstring += i
# else:
# self.cellstring += ' No cell description'
self.cellstring += ' \t'
self.doProtocols(os.path.join(slice, cell))
DataManager.cleanup() # clean up after each cell
del dh
gc.collect()
def doSlices(self, day):
"""
process all of the slices for a given day
:param day:
:return nothing:
"""
allfiles = os.listdir(day)
slicetype = re.compile("(slice\_)(\d{3,3})")
slices = []
for thisfile in allfiles:
m = slicetype.match(thisfile)
if m is None:
continue
if len(m.groups()) == 2:
slices.append(thisfile)
for slice in slices:
self.slicestring = '%s \t' % (slice)
dh = DataManager.getDirHandle(os.path.join(day, slice),
create=False)
sl = self.dataModel.getSliceInfo(dh)
# if sl is not None and 'description' in sl.keys() and len(sl['description']) > 0:
# l = self.twd['slice'].wrap(sl['description'])
# for i in l:
# self.slicestring += i
# else:
# self.slicestring += ' No slice description'
# self.slicestring += '\t'
if sl is not None and 'notes' in sl.keys() and len(
sl['notes']) > 0:
l = self.tw['slice'].wrap(sl['notes'])
for i in l:
i = i.replace(
'\t', ' '
) # clean out tabs so printed formatting is not confused
self.slicestring += i
else:
self.slicestring += ' No slice notes'
self.slicestring += ' \t'
self.doCells(os.path.join(day, slice))
DataManager.cleanup()
del dh
gc.collect()
def addSiteContents(siteNWBs, filesToInclude, slicePath, siteName):
""" Add site specific entries to the NWB file """
sitePath = os.path.join(slicePath, siteName)
siteIndex = os.path.join(sitePath, ".index")
if len(siteNWBs) != 1:
print "Expected exactly one NWB file belonging to site folder %s, skipping it." % sitePath
return 1
sitePXP = glob.glob(os.path.join(sitePath, '*.pxp'))[0]
addDataSource(siteNWBs, sitePXP)
dh = adm.getHandle(sitePath)
dh.checkIndex()
data = encodeAsJSONString(dh["."].info())
appendMiscFileToNWB(siteNWBs, "%s_index_meta" % siteName, data)
appendMiscFileToNWB(siteNWBs, "%s_index" % siteName,
getFileContents(siteIndex))
for k in dh.ls():
if not dh.isManaged(k):
continue
path = os.path.join(sitePath, k)
if os.path.isdir(path):
raise NameError("Unexpected folder \"%s\" in \"%s\"." %
(k, sitePath))
elif os.path.isfile(path): # check if we need to handle it
if fileShouldBeSkipped(path, filesToInclude):
continue
filedesc = encodeAsJSONString(dh[k].info())
if path.endswith(".tif"):
appendImageFileToNWB(siteNWBs, path, filedesc)
elif path.endswith(".ma"):
appendMAFile(siteNWBs, path, filedesc)
elif path.endswith(".log"):
appendPseudoYamlLog(siteNWBs, path, k, filedesc)
else:
raise NameError(
"Unexpected file type \"%s\" in index \"%s\"." %
(k, siteIndex))
else:
raise NameError("Unexpected key \"%s\" in index \"%s\"." %
(k, siteIndex))
def __init__(self, atlas, host):
Atlas.AtlasCtrlWidget.__init__(self, atlas, host)
self.atlasDir = os.path.split(os.path.abspath(__file__))[0]
## add ThalamocorticalMarker to canvas
fh = DataManager.getHandle(os.path.join(self.atlasDir, 'images', 'ThalamocorticalMarker.svg'))
self.canvas.addFile(fh, pos=(-0.001283, -0.000205), scale=[3.78e-6, 3.78e-6], index=0, movable=False, z=10000)
## add CortexROI
self.roi = CortexROI([-1e-3, 0])
self.canvas.addGraphicsItem(self.roi, pos=(-1e-3, 1e-3), scale=[1e-3, 1e-3], name='CortexROI', movable=False)
self.roi.sigRegionChangeFinished.connect(self.roiChanged)
def buildCombinedNWBInternal(basepath, siteNWBs, filesToInclude):
""" NOT FOR PUBLIC USE """
# we have three types of keys in the main index file
# ---------------------------------------------------------------------
# '.' | common description of the experiment | (unique)
# '$existingFile' | log file of the experiment | (multiple)
# '$existingFolder' | different slices for each experiment | (multiple)
dh = adm.getHandle(basepath)
dh.checkIndex()
data = encodeAsJSONString(dh["."].info())
appendMiscFileToNWB(siteNWBs, basename="main_index_meta", content=data)
logfile = os.path.join(basepath, '.index')
appendMiscFileToNWB(siteNWBs,
basename="main_index",
content=getFileContents(logfile))
for elem in siteNWBs:
os.remove(deriveOutputNWB(elem))
for k in dh.ls():
if not dh.isManaged(k):
continue
path = os.path.abspath(os.path.join(basepath, k))
if os.path.isdir(path): # slice folder
addSliceContents(siteNWBs, filesToInclude, basepath, k)
elif os.path.isfile(path): # main log file
data = encodeAsJSONString(dh[k].info())
appendMiscFileToNWB(siteNWBs,
basename="main_logfile_meta",
content=data)
appendMiscFileToNWB(siteNWBs,
basename="main_logfile",
content=getFileContents(path))
else:
raise NameError("Unexpected key \"%s\" in index \"%s\"" %
(k, logfile))
combinedNWBs = []
for elem in siteNWBs:
combinedNWBs.append(deriveOutputNWB(elem))
return combinedNWBs
def addSliceContents(siteNWBs, filesToInclude, basepath, sliceName):
""" Add entries to the slice specific NWB file """
slicePath = os.path.join(basepath, sliceName)
sliceIndex = os.path.join(slicePath, ".index")
sliceNWBs = [elem for elem in siteNWBs if elem.startswith(slicePath)]
if len(sliceNWBs) == 0:
#print "No NWB files belong to slice folder %s, skipping it." % slicePath
return 1
dh = adm.getHandle(slicePath)
dh.checkIndex()
data = encodeAsJSONString(dh["."].info())
appendMiscFileToNWB(siteNWBs, "%s_index_meta" % sliceName, data)
appendMiscFileToNWB(siteNWBs, "%s_index" % sliceName,
getFileContents(sliceIndex))
for k in dh.ls():
if not dh.isManaged(k):
continue
path = os.path.abspath(os.path.join(slicePath, k))
filedesc = encodeAsJSONString(dh[k].info())
if os.path.isdir(path): # site folder
appendMiscFileToNWB(sliceNWBs, "%s_%s" % (sliceName, k), filedesc)
addSiteContents(sliceNWBs, filesToInclude, slicePath, k)
elif os.path.isfile(path): # check if we need to handle it
if fileShouldBeSkipped(path, filesToInclude):
continue
if path.endswith(".tif"):
appendImageFileToNWB(sliceNWBs, path, filedesc)
elif path.endswith(".ma"):
appendMAFile(sliceNWBs, path, filedesc)
else:
raise NameError("Unexpected file type \"%s\" in index \"%s\"" %
(k, sliceIndex))
else:
raise NameError("Unexpected key \"%s\" in index \"%s\"" %
(k, sliceIndex))
def addSiteContents(siteNWBs, filesToInclude, slicePath, siteName):
""" Add site specific entries to the NWB file """
sitePath = os.path.join(slicePath, siteName)
siteIndex = os.path.join(sitePath, ".index")
if len(siteNWBs) != 1:
print "Expected exactly one NWB file belonging to site folder %s, skipping it." % sitePath
return 1
addDataSource(siteNWBs)
dh = adm.getHandle(sitePath)
dh.checkIndex()
data = encodeAsJSONString(dh["."].info())
appendMiscFileToNWB(siteNWBs, "%s_index_meta" % siteName, data)
appendMiscFileToNWB(siteNWBs, "%s_index" % siteName, getFileContents(siteIndex))
for k in dh.ls():
if not dh.isManaged(k):
continue
path = os.path.join(sitePath, k)
if os.path.isdir(path):
raise NameError("Unexpected folder \"%s\" in \"%s\"." % (k, sitePath))
elif os.path.isfile(path): # check if we need to handle it
if fileShouldBeSkipped(path, filesToInclude):
continue
filedesc = encodeAsJSONString(dh[k].info())
if path.endswith(".tif"):
appendImageFileToNWB(siteNWBs, path, filedesc)
elif path.endswith(".ma"):
appendMAFile(siteNWBs, path, filedesc)
elif path.endswith(".log"):
appendPseudoYamlLog(siteNWBs, path, k, filedesc)
else:
raise NameError("Unexpected file type \"%s\" in index \"%s\"." % (k, siteIndex))
else:
raise NameError("Unexpected key \"%s\" in index \"%s\"." % (k, siteIndex))
def uiChanged(self):
for item in self.images:
self.canvas.removeItem(item)
self.images = []
state = self.stateGroup.state()
slice = state['slicePlaneCombo']
hem = state['hemisphereCombo']
#flip = state['flipCheck']
imgs = self.slicePlanes[slice]
atlasDir = os.path.split(os.path.abspath(__file__))[0]
for imgFile, scale, pos in imgs:
fh = DataManager.getHandle(os.path.join(atlasDir, 'images', imgFile))
item = self.canvas.addFile(fh, pos=pos, scale=[scale,scale], index=0, movable=False, z=10000)
#item = self.canvas.addFile(fh, index=0, movable=False)
self.images.append(item)
def load_stimulus(self, recording):
"""Return an instance of stimuli.Stimulus"""
#### I don't know whether I should try to parse this from metadata, or just find square pulses in the command waveform.
### I think finding square pulses would be simpler, but makes the assumption that pulses are square. Which is probably usually true.
### what if I check the wavegenerator widget data for the function name (pulse) and then findSquarepulses, or raise an exception if it's a different function?
if isinstance(recording, PatchClampRecording):
items = []
fh = DataManager.getFileHandle(recording.meta['file_name'])
seqDir = PatchEPhys.getParent(fh, 'ProtocolSequence')
if seqDir is not None:
dev_info = seqDir.info()['devices'][recording.device_id]
if dev_info['mode'].lower() == 'vc':
units = 'V'
elif dev_info['mode'].lower() == 'ic':
units = 'A'
else:
units = None
if dev_info['holdingCheck']:
items.append(stimuli.Offset(dev_info['holdingSpin']))
stim_pulses = PatchEPhys.getStimParams(fh)
for p in stim_pulses:
if p['function_type'] == 'pulse':
items.append(
stimuli.SquarePulse(p['start'], p['length'],
p['amplitude']))
elif p['function_type'] == 'pulseTrain':
items.append(
stimuli.SquarePulseTrain(p['start'],
p['pulse_number'],
p['length'],
p['amplitude'],
p['period']))
desc = seqDir.shortName()[:-4]
return stimuli.Stimulus(desc, items=items, units=units)
else:
raise Exception('not implemented yet')
def addSliceContents(siteNWBs, filesToInclude, basepath, sliceName):
""" Add entries to the slice specific NWB file """
slicePath = os.path.join(basepath, sliceName)
sliceIndex = os.path.join(slicePath, ".index")
sliceNWBs = [elem for elem in siteNWBs if elem.startswith(slicePath)]
if len(sliceNWBs) == 0:
#print "No NWB files belong to slice folder %s, skipping it." % slicePath
return 1
dh = adm.getHandle(slicePath)
dh.checkIndex()
data = encodeAsJSONString(dh["."].info())
appendMiscFileToNWB(siteNWBs, "%s_index_meta" % sliceName, data)
appendMiscFileToNWB(siteNWBs, "%s_index" % sliceName, getFileContents(sliceIndex))
for k in dh.ls():
if not dh.isManaged(k):
continue
path = os.path.abspath(os.path.join(slicePath, k))
filedesc = encodeAsJSONString(dh[k].info())
if os.path.isdir(path): # site folder
appendMiscFileToNWB(sliceNWBs, "%s_%s" % (sliceName, k), filedesc)
addSiteContents(sliceNWBs, filesToInclude, slicePath, k)
elif os.path.isfile(path): # check if we need to handle it
if fileShouldBeSkipped(path, filesToInclude):
continue
if path.endswith(".tif"):
appendImageFileToNWB(sliceNWBs, path, filedesc)
elif path.endswith(".ma"):
appendMAFile(sliceNWBs, path, filedesc)
else:
raise NameError("Unexpected file type \"%s\" in index \"%s\"" % (k, sliceIndex))
else:
raise NameError("Unexpected key \"%s\" in index \"%s\"" % (k, sliceIndex))
def buildCombinedNWBInternal(basepath, siteNWBs, filesToInclude):
""" NOT FOR PUBLIC USE """
# we have three types of keys in the main index file
# ---------------------------------------------------------------------
# '.' | common description of the experiment | (unique)
# '$existingFile' | log file of the experiment | (multiple)
# '$existingFolder' | different slices for each experiment | (multiple)
dh = adm.getHandle(basepath)
dh.checkIndex()
data = encodeAsJSONString(dh["."].info())
appendMiscFileToNWB(siteNWBs, basename = "main_index_meta", content = data)
logfile = os.path.join(basepath, '.index')
appendMiscFileToNWB(siteNWBs, basename = "main_index", content = getFileContents(logfile))
for elem in siteNWBs:
os.remove(deriveOutputNWB(elem))
for k in dh.ls():
if not dh.isManaged(k):
continue
path = os.path.abspath(os.path.join(basepath, k))
if os.path.isdir(path): # slice folder
addSliceContents(siteNWBs, filesToInclude, basepath, k)
elif os.path.isfile(path): # main log file
data = encodeAsJSONString(dh[k].info())
appendMiscFileToNWB(siteNWBs, basename = "main_logfile_meta", content = data)
appendMiscFileToNWB(siteNWBs, basename = "main_logfile", content = getFileContents(path))
else:
raise NameError("Unexpected key \"%s\" in index \"%s\"" % (k, logfile))
combinedNWBs = []
for elem in siteNWBs:
combinedNWBs.append(deriveOutputNWB(elem))
return combinedNWBs
def baseDir(self):
"""Return a dirHandle for the base directory used for all file names in the database."""
if self._baseDir is None:
dirName = self.ctrlParam('BaseDirectory')
self._baseDir = DataManager.getHandle(dirName)
return self._baseDir
def saveStateFile(self, filename):
dh = DataManager.getDirHandle(os.path.dirname(filename))
state = self.saveState(relativeTo=dh)
json.dump(state, open(filename, 'w'), indent=4, cls=Encoder)
def dh(self):
if self._dh is None:
self._dh = DataManager.getDirHandle(self._filepath)
return self._dh
import os, sys
d = os.path.split(os.path.abspath(__file__))[0]
d1 = os.path.split(d)[0]
d2 = os.path.split(d1)[0]
sys.path.extend([d1, d2])
from acq4.util.DirTreeWidget import *
from DirTreeLoader import *
from acq4.util.DataManager import *
from PyQt4 import QtCore, QtGui
app = QtGui.QApplication([])
dm = DataManager()
dh = dm.getDirHandle(d)['testDir']
class Loader(DirTreeLoader):
def new(self):
print "NEW"
return True
def save(self, fh):
open(fh.name(), 'w').write("SAVED")
print "SAVED"
return True
def load(self, fh):
print "LOADED:", open(fh.name()).read()
def doProtocols(self, cell):
"""
process all of the protocols for a given cell
:param cell:
:return nothing:
"""
allfiles = os.listdir(cell)
#celltype = re.compile("(Cell_)(\d{3,3})")
protocols = []
nonprotocols = []
anyprotocols = False
images = [] # tiff
stacks2p = []
images2p = []
videos = []
endmatch = re.compile(
"[\_(\d{3,3})]$") # look for _lmn at end of directory name
for thisfile in allfiles:
if os.path.isdir(os.path.join(cell, thisfile)):
protocols.append(thisfile)
else:
nonprotocols.append(thisfile)
self.protocolstring = ''
if self.InvestigateProtocols is True:
self.summarystring = 'NaN \t' * 6
for np, protocol in enumerate(protocols):
dh = DataManager.getDirHandle(os.path.join(cell, protocol),
create=False)
if np == 0:
self.cell_summary(dh)
if self.monitor:
print 'Investigating Protocol: %s', dh.name()
dirs = dh.subDirs()
protocolok = True # assume that protocol is ok
modes = []
nexpected = len(
dirs
) # acq4 writes dirs before, so this is the expected fill
ncomplete = 0 # count number actually done
clampDevices = self.dataModel.getClampDeviceNames(dh)
# must handle multiple data formats, even in one experiment...
if clampDevices is not None:
data_mode = dh.info()['devices'][clampDevices[0]][
'mode'] # get mode from top of protocol information
else: # try to set a data mode indirectly
if 'devices' not in dh.info().keys():
protocolok = False # can't parse protocol device...
continue
devices = dh.info()['devices'].keys(
) # try to get clamp devices from another location
#print dir(self.dataModel)
for kc in self.dataModel.knownClampNames():
if kc in devices:
clampDevices = [kc]
try:
data_mode = dh.info()['devices'][
clampDevices[0]]['mode']
except:
data_mode = 'Unknown'
# protocolok = False
# print '<<cannot read protocol data mode>>'
if data_mode not in modes:
modes.append(data_mode)
for i, directory_name in enumerate(
dirs
): # dirs has the names of the runs within the protocol
data_dir_handle = dh[
directory_name] # get the directory within the protocol
try:
data_file_handle = self.dataModel.getClampFile(
data_dir_handle) # get pointer to clamp data
except:
data_file_handle = None
if data_file_handle is not None: # no clamp file found - skip
ncomplete += 1
# Check if there is no clamp file for this iteration of the protocol
# Usually this indicates that the protocol was stopped early.
# data_file = data_file_handle.read()
try:
self.holding = self.dataModel.getClampHoldingLevel(
data_file_handle)
except:
self.holding = 0.
try:
self.amp_settings = self.dataModel.getWCCompSettings(
data_file_handle)
except:
self.amp_settings = None
#raise ValueError('complete = %d when failed' % ncomplete)
#else:
# break # do not keep looking if the file is not found
DataManager.cleanup() # close all opened files
# del dh
gc.collect(
) # and force garbage collection of freed objects inside the loop
if modes == []:
modes = ['Unknown mode']
if protocolok and ncomplete == nexpected: # accumulate protocols
self.protocolstring += '[{:<s}: {:s} {:d}], '.format(
protocol, modes[0][0], ncomplete)
anyprotocols = True # indicate that ANY protocol ran to completion
else:
if self.reportIncompleteProtocols:
self.protocolstring += '[{:<s}, ({:s}, {:d}/{:d}, Incomplete)], '.format(
protocol, modes[0][0], ncomplete, nexpected)
DataManager.cleanup()
del dh
gc.collect()
else:
self.protocolstring += 'Protocols: '
anyprotocols = True
prots = {}
for protocol in protocols:
m = endmatch.search(protocol)
if m is not None:
p = protocol[:-4]
else:
p = protocol
if p not in prots.keys():
prots[p] = 1
else:
prots[p] += 1
if len(prots.keys()) > 0:
self.protocolstring += '['
for p in prots.keys():
self.protocolstring += '{:<s}({:<d}), '.format(p, prots[p])
self.protocolstring += ']'
else:
self.protocolstring = '<No protocols found>'
self.protocolstring += ' \t'
for thisfile in nonprotocols:
# if os.path.isdir(os.path.join(cell, thisfile)): # skip protocols
# continue
x = self.img_re.match(thisfile) # look for image files
if x is not None:
images.append(thisfile)
x = self.s2p_re.match(thisfile) # two photon stacks
if x is not None:
stacks2p.append(thisfile)
x = self.i2p_re.match(thisfile) # simple two photon images
if x is not None:
images2p.append(thisfile)
x = self.video_re.match(thisfile) # video images
if x is not None:
videos.append(thisfile)
self.imagestring = ''
if len(images) > 0:
self.imagestring += 'Images: %d ' % len(images)
if len(stacks2p) > 0:
self.imagestring += '2pStacks: %d ' % len(stacks2p)
if len(images2p) > 0:
self.imagestring += '2pImages: %d ' % len(images2p)
if len(videos) > 0:
self.imagestring += 'Videos: %d' % len(videos)
if len(images) + len(stacks2p) + len(images2p) + len(videos) == 0:
self.imagestring = 'No Images or Videos'
if anyprotocols:
ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + self.protocolstring + self.imagestring + ' \t'
else:
ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + '<No complete protocols> \t' + self.imagestring + ' \t'
self.outputString(ostring)
def baseDir(self):
"""Return a dirHandle for the base directory used for all file names in the database."""
if self._baseDir is None:
dirName = self.ctrlParam('BaseDirectory')
self._baseDir = DataManager.getHandle(dirName)
return self._baseDir
def getSummary(self):
"""
getSummary is the entry point for scanning through all the data files in a given directory,
returning information about those within the date range, with details as specified by the options
"""
allfiles = os.listdir(self.basedir)
days = []
for thisfile in allfiles:
m = self.daytype.match(thisfile)
if m == '.DS_Store':
continue
if m is None:
# print 'Top level file %s is incorrectly placed ' % thisfile
continue # no match
if len(m.groups()) >= 3: # perfect match
# print m.groups()
idl = [int(d) for d in m.groups()]
id = idl[0] * 1e4 + idl[1] * 1e2 + idl[2]
# print 'id: ', id
# print 'minday: ', minday
if self.daylist is None:
if id >= self.minday and id <= self.maxday:
days.append(thisfile) # was [0:10]
else:
#print 'using daylist, thisfile: ', thisfile[0:10]
#print 'daylist: ', daylist
if thisfile[0:10] in self.daylist:
days.append(thisfile)
if self.monitor:
print 'Days reported: ', days
for day in days:
if self.monitor:
print 'processing day: %s' % day
self.daystring = '%s \t' % (day)
dh = DataManager.getDirHandle(os.path.join(self.basedir, day),
create=False)
dx = self.dataModel.getDayInfo(dh)
if dx is not None and 'description' in dx.keys() and len(
dx['description']) > 0:
l = self.twd['day'].wrap(dx['description'])
for i in l:
i = i.replace(
'\t', ' '
) # clean out tabs so printed formatting is not confused
self.daystring += i
else:
self.daystring += ' [no description]'
self.daystring += ' \t'
if dx is not None and 'notes' in dx.keys() and len(
dx['notes']) > 0:
l = self.tw['day'].wrap(dx['notes'])
for i in l:
i = i.replace(
'\t', ' '
) # clean out tabs so printed formatting is not confused
self.daystring += i
else:
self.daystring += ' [no notes]'
self.daystring += ' \t'
self.doSlices(os.path.join(self.basedir, day))
os.closerange(8, 65535) # close all files in each iteration
gc.collect()
# -*- coding: utf-8 -*-
import os, sys
d = os.path.split(os.path.abspath(__file__))[0]
d1 = os.path.split(d)[0]
d2 = os.path.split(d1)[0]
sys.path.extend([d1, d2])
from acq4.util.DirTreeWidget import *
from DirTreeLoader import *
from acq4.util.DataManager import *
from PyQt4 import QtCore,QtGui
app = QtGui.QApplication([])
dm = DataManager()
dh = dm.getDirHandle(d)['testDir']
class Loader(DirTreeLoader):
def new(self):
print "NEW"
return True
def save(self, fh):
open(fh.name(), 'w').write("SAVED")
print "SAVED"
return True
def load(self, fh):
print "LOADED:", open(fh.name()).read()
def run_script(self):
"""
Run a script, doing all of the requested analysis
:return:
"""
if self.script['testfiles']:
return
# settext = self.scripts_form.PSPReversal_ScriptResults_text.setPlainText
# apptext = self.scripts_form.PSPReversal_ScriptResults_text.appendPlainText
self.textout = ('\nScript File: {:<32s}\n'.format(self.script_name))
# settext(self.textout)
script_header = True # reset the table to a print new header for each cell
trailingchars = [
c for c in map(chr, range(97, 123))
] # trailing chars used to identify different parts of a cell's data
self.dataManager().setBaseDir(self.script['directory'])
ordered = sorted(self.script['Cells'].keys()
) # order the analysis by date/slice/cell
prog1 = ProgressDialog("Script Processing..", 0, len(ordered))
ncell = len(ordered)
for nc, cell in enumerate(ordered):
if prog1.wasCanceled():
break
presetDict = {}
thiscell = self.script['Cells'][cell]
#print 'processing cell: %s' % thiscell
if thiscell['include'] is False: # skip this cell
try:
print('Skipped: %s, reason:%s' %
(cell, thiscell['reason']))
except:
raise ValueError(
'cell %s has no tag "reason" but "include" is False' %
cell)
continue
sortedkeys = sorted(thiscell['choice'].keys()
) # sort by order of recording (# on protocol)
prog1.setValue(nc / ncell)
# prog2 = ProgressDialog("Cell Processing..%s" , 0, len(sortedkeys)):
for p in sortedkeys:
if thiscell['choice'][p] not in self.script[
'datafilter']: # pick out steady-state conditions
print('p: %s not in data: ' % (thiscell['choice'][p]),
self.script['datafilter'])
continue
# print 'working on %s' % thiscell['choice'][p]
pr = self.script[
'protocol'] + '_' + p # add the underscore here
if cell[-1] in trailingchars: # check last letter - if not a number clip it
cell_file = cell[:-1]
else:
cell_file = cell
fn = os.path.join(cell_file, pr)
#dm_selected_file = self.dataManager().selectedFile().name()
dm_selected_file = self.script['directory']
fullpath = os.path.join(dm_selected_file, fn)
file_ok = os.path.exists(fullpath)
if not file_ok: # get the directory handle and take it from there
print('File is not ok: %s' % fullpath)
continue
m = thiscell['choice'][p] # get the tag for the manipulation
presetDict['Choices'] = thiscell['choice'][p]
if 'genotype' in thiscell.keys():
presetDict['Genotype'] = thiscell['genotype']
else:
presetDict['Genotype'] = 'Unknown'
if 'Celltype' in thiscell.keys():
presetDict['Celltype'] = thiscell['Celltype']
else:
presetDict['Celltype'] = 'Unknown'
if 'spikethresh' in thiscell.keys():
presetDict['SpikeThreshold'] = thiscell['spikethresh']
if 'bridgeCorrection' in thiscell.keys():
presetDict['bridgeCorrection'] = thiscell[
'bridgeCorrection']
else:
presetDict['bridgeCorrection'] = None
dh = self.dataManager().manager.dirHandle(fullpath)
if not self.loadFile(
[dh], analyze=False, bridge=presetDict['bridgeCorrection']
): # note: must pass a list of dh; don't let analyisis run at end
print('Failed to load requested file: ', fullpath)
continue # skip bad sets of records...
if 'datamode' in thiscell.keys():
self.clamps.data_mode = thiscell['datamode']
self.auto_updater = False
self.get_script_analysisPars(self.script, thiscell)
self.analysis(
presets=presetDict) # call the caller's analysis routine
if 'addtoDB' in self.script.keys():
if self.script[
'addtoDB'] is True and self.dbUpdate is not None:
self.dbUpdate() # call routine in parent
ptxt = self.printAnalysis(printnow=False,
script_header=script_header,
copytoclipboard=False)
self.textout += ptxt + '\n'
script_header = False
DataManager.cleanup()
del dh
gc.collect()
print(self.textout)
self.auto_updater = True # restore function