def test_datamanager():
rh = dm.getDirHandle(root)
# check handles are cached
rh2 = dm.getDirHandle(root)
assert rh is rh2
# test meta info is stored and reloaded correctly
rh.setInfo({'test_int': 1, 'test_str': 'xxx'})
assert rh.info()['test_int'] == 1
assert rh.info()['test_str'] == 'xxx'
# Create a subdir with meta info
d1 = rh.mkdir('subdir', info={'a': 'b'})
assert d1.info()['a'] == 'b'
assert d1.shortName() == 'subdir'
assert d1.name() == os.path.join(rh.name(), 'subdir')
# Create a DirTreeWidget; check that the contents are correct.
dw = DirTreeWidget(baseDirHandle=rh)
assert dw.topLevelItemCount() == 1
item = dw.topLevelItem(0)
assert item.text(0) == d1.shortName()
assert item.handle is d1
assert item.childCount() == 0
# Create a subdir and update the tree widget
d2 = d1.mkdir('subdir2')
dw.rebuildChildren(item)
assert item.childCount() == 1
# test _getTree
d3 = rh.mkdir('subdir3')
assert d3.name() not in dm.dm._getTree(d1.name())
assert d2.name() in dm.dm._getTree(d1.name())
#
# root
# + subdir
# | + subdir2
# + subdir3
#
assert d1.name(relativeTo=rh) == 'subdir'
assert d2.name(relativeTo=rh) == os.path.join('subdir', 'subdir2')
assert d2.name(relativeTo=d1) == 'subdir2'
assert d2.name(relativeTo=d2) == ''
assert d1.name(relativeTo=d2) == '..'
assert rh.name(relativeTo=d2) == os.path.join('..', '..')
assert d3.name(relativeTo=d2) == os.path.join('..', '..', 'subdir3')
assert d2.name(relativeTo=d3) == os.path.join('..', 'subdir', 'subdir2')
# rename subdir from tree widget
item.setText(0, 'subdir_renamed')
assert d1.shortName() == 'subdir_renamed'
# delete subdir
d1.delete()
dw.rebuildTree()
assert dw.topLevelItemCount() == 1
def __init__(self, baseDir, sortMode='alpha', create=False, *args):
QtGui.QWidget.__init__(self, *args)
self.ui = Ui_Form()
self.ui.setupUi(self)
if isinstance(baseDir, basestring):
baseDir = DataManager.getDirHandle(baseDir, create=create)
self.baseDir = baseDir
self.currentFile = None
#self.fileTree = DirTreeModel(baseDir)
#self.ui.fileTree.setModel(self.fileTree)
self.ui.fileTree.setSortMode(sortMode)
self.ui.fileTree.setBaseDirHandle(baseDir)
self.deleteState = 0
self.ui.deleteBtn.focusOutEvent = self.delBtnLostFocus
self.ui.newBtn.clicked.connect(self.newClicked)
self.ui.newDirBtn.clicked.connect(self.newDirClicked)
self.ui.saveBtn.clicked.connect(self.saveClicked)
self.ui.loadBtn.clicked.connect(self.loadClicked)
self.ui.saveAsBtn.clicked.connect(self.saveAsClicked)
self.ui.deleteBtn.clicked.connect(self.deleteClicked)
self.ui.fileTree.itemDoubleClicked.connect(self.loadClicked)
def __init__(self, baseDir, sortMode='alpha', create=False, *args):
QtGui.QWidget.__init__(self, *args)
self.ui = Ui_Form()
self.ui.setupUi(self)
if isinstance(baseDir, basestring):
baseDir = DataManager.getDirHandle(baseDir, create=create)
self.baseDir = baseDir
self.currentFile = None
#self.fileTree = DirTreeModel(baseDir)
#self.ui.fileTree.setModel(self.fileTree)
self.ui.fileTree.setSortMode(sortMode)
self.ui.fileTree.setBaseDirHandle(baseDir)
self.deleteState = 0
self.ui.deleteBtn.focusOutEvent = self.delBtnLostFocus
self.ui.newBtn.clicked.connect(self.newClicked)
self.ui.newDirBtn.clicked.connect(self.newDirClicked)
self.ui.saveBtn.clicked.connect(self.saveClicked)
self.ui.loadBtn.clicked.connect(self.loadClicked)
self.ui.saveAsBtn.clicked.connect(self.saveAsClicked)
self.ui.deleteBtn.clicked.connect(self.deleteClicked)
self.ui.fileTree.itemDoubleClicked.connect(self.loadClicked)
def doCells(self, slice):
"""
process all of the cells from a slice
:param slice:
:return nothing:
"""
allfiles = os.listdir(slice)
celltype = re.compile("(cell_)(\d{3,3})")
cells = []
for thisfile in allfiles:
m = celltype.match(thisfile)
if m is None:
continue
if len(m.groups()) == 2:
cells.append(thisfile)
for cell in cells:
self.cellstring = '%s\t' % (cell)
dh = DataManager.getDirHandle(os.path.join(slice, cell),
create=False)
cl = self.dataModel.getCellInfo(dh)
if cl is not None and 'notes' in cl.keys() and len(
cl['notes']) > 0:
l = self.tw['cell'].wrap(cl['notes'])
for i in l:
i = i.replace(
'\t', ' '
) # clean out tabs so printed formatting is not confused
self.cellstring += i
else:
self.cellstring += ' No cell notes'
self.cellstring += ' \t'
self.cell_summary(dh)
# if cl is not None and 'description' in cl.keys() and len(cl['description']) > 0:
# l = self.twd['cell'].wrap(cl['description'])
# for i in l:
# self.cellstring += i
# else:
# self.cellstring += ' No cell description'
self.cellstring += ' \t'
self.doProtocols(os.path.join(slice, cell))
DataManager.cleanup() # clean up after each cell
del dh
gc.collect()
def doSlices(self, day):
"""
process all of the slices for a given day
:param day:
:return nothing:
"""
allfiles = os.listdir(day)
slicetype = re.compile("(slice\_)(\d{3,3})")
slices = []
for thisfile in allfiles:
m = slicetype.match(thisfile)
if m is None:
continue
if len(m.groups()) == 2:
slices.append(thisfile)
for slice in slices:
self.slicestring = '%s \t' % (slice)
dh = DataManager.getDirHandle(os.path.join(day, slice),
create=False)
sl = self.dataModel.getSliceInfo(dh)
# if sl is not None and 'description' in sl.keys() and len(sl['description']) > 0:
# l = self.twd['slice'].wrap(sl['description'])
# for i in l:
# self.slicestring += i
# else:
# self.slicestring += ' No slice description'
# self.slicestring += '\t'
if sl is not None and 'notes' in sl.keys() and len(
sl['notes']) > 0:
l = self.tw['slice'].wrap(sl['notes'])
for i in l:
i = i.replace(
'\t', ' '
) # clean out tabs so printed formatting is not confused
self.slicestring += i
else:
self.slicestring += ' No slice notes'
self.slicestring += ' \t'
self.doCells(os.path.join(day, slice))
DataManager.cleanup()
del dh
gc.collect()
def saveStateFile(self, filename):
dh = DataManager.getDirHandle(os.path.dirname(filename))
state = self.saveState(relativeTo=dh)
json.dump(state, open(filename, 'w'), indent=4, cls=Encoder)
import os, sys
d = os.path.split(os.path.abspath(__file__))[0]
d1 = os.path.split(d)[0]
d2 = os.path.split(d1)[0]
sys.path.extend([d1, d2])
from acq4.util.DirTreeWidget import *
from DirTreeLoader import *
from acq4.util.DataManager import *
from PyQt4 import QtCore, QtGui
app = QtGui.QApplication([])
dm = DataManager()
dh = dm.getDirHandle(d)['testDir']
class Loader(DirTreeLoader):
def new(self):
print "NEW"
return True
def save(self, fh):
open(fh.name(), 'w').write("SAVED")
print "SAVED"
return True
def load(self, fh):
print "LOADED:", open(fh.name()).read()
return True
def doProtocols(self, cell):
"""
process all of the protocols for a given cell
:param cell:
:return nothing:
"""
allfiles = os.listdir(cell)
#celltype = re.compile("(Cell_)(\d{3,3})")
protocols = []
nonprotocols = []
anyprotocols = False
images = [] # tiff
stacks2p = []
images2p = []
videos = []
endmatch = re.compile(
"[\_(\d{3,3})]$") # look for _lmn at end of directory name
for thisfile in allfiles:
if os.path.isdir(os.path.join(cell, thisfile)):
protocols.append(thisfile)
else:
nonprotocols.append(thisfile)
self.protocolstring = ''
if self.InvestigateProtocols is True:
self.summarystring = 'NaN \t' * 6
for np, protocol in enumerate(protocols):
dh = DataManager.getDirHandle(os.path.join(cell, protocol),
create=False)
if np == 0:
self.cell_summary(dh)
if self.monitor:
print 'Investigating Protocol: %s', dh.name()
dirs = dh.subDirs()
protocolok = True # assume that protocol is ok
modes = []
nexpected = len(
dirs
) # acq4 writes dirs before, so this is the expected fill
ncomplete = 0 # count number actually done
clampDevices = self.dataModel.getClampDeviceNames(dh)
# must handle multiple data formats, even in one experiment...
if clampDevices is not None:
data_mode = dh.info()['devices'][clampDevices[0]][
'mode'] # get mode from top of protocol information
else: # try to set a data mode indirectly
if 'devices' not in dh.info().keys():
protocolok = False # can't parse protocol device...
continue
devices = dh.info()['devices'].keys(
) # try to get clamp devices from another location
#print dir(self.dataModel)
for kc in self.dataModel.knownClampNames():
if kc in devices:
clampDevices = [kc]
try:
data_mode = dh.info()['devices'][
clampDevices[0]]['mode']
except:
data_mode = 'Unknown'
# protocolok = False
# print '<<cannot read protocol data mode>>'
if data_mode not in modes:
modes.append(data_mode)
for i, directory_name in enumerate(
dirs
): # dirs has the names of the runs within the protocol
data_dir_handle = dh[
directory_name] # get the directory within the protocol
try:
data_file_handle = self.dataModel.getClampFile(
data_dir_handle) # get pointer to clamp data
except:
data_file_handle = None
if data_file_handle is not None: # no clamp file found - skip
ncomplete += 1
# Check if there is no clamp file for this iteration of the protocol
# Usually this indicates that the protocol was stopped early.
# data_file = data_file_handle.read()
try:
self.holding = self.dataModel.getClampHoldingLevel(
data_file_handle)
except:
self.holding = 0.
try:
self.amp_settings = self.dataModel.getWCCompSettings(
data_file_handle)
except:
self.amp_settings = None
#raise ValueError('complete = %d when failed' % ncomplete)
#else:
# break # do not keep looking if the file is not found
DataManager.cleanup() # close all opened files
# del dh
gc.collect(
) # and force garbage collection of freed objects inside the loop
if modes == []:
modes = ['Unknown mode']
if protocolok and ncomplete == nexpected: # accumulate protocols
self.protocolstring += '[{:<s}: {:s} {:d}], '.format(
protocol, modes[0][0], ncomplete)
anyprotocols = True # indicate that ANY protocol ran to completion
else:
if self.reportIncompleteProtocols:
self.protocolstring += '[{:<s}, ({:s}, {:d}/{:d}, Incomplete)], '.format(
protocol, modes[0][0], ncomplete, nexpected)
DataManager.cleanup()
del dh
gc.collect()
else:
self.protocolstring += 'Protocols: '
anyprotocols = True
prots = {}
for protocol in protocols:
m = endmatch.search(protocol)
if m is not None:
p = protocol[:-4]
else:
p = protocol
if p not in prots.keys():
prots[p] = 1
else:
prots[p] += 1
if len(prots.keys()) > 0:
self.protocolstring += '['
for p in prots.keys():
self.protocolstring += '{:<s}({:<d}), '.format(p, prots[p])
self.protocolstring += ']'
else:
self.protocolstring = '<No protocols found>'
self.protocolstring += ' \t'
for thisfile in nonprotocols:
# if os.path.isdir(os.path.join(cell, thisfile)): # skip protocols
# continue
x = self.img_re.match(thisfile) # look for image files
if x is not None:
images.append(thisfile)
x = self.s2p_re.match(thisfile) # two photon stacks
if x is not None:
stacks2p.append(thisfile)
x = self.i2p_re.match(thisfile) # simple two photon images
if x is not None:
images2p.append(thisfile)
x = self.video_re.match(thisfile) # video images
if x is not None:
videos.append(thisfile)
self.imagestring = ''
if len(images) > 0:
self.imagestring += 'Images: %d ' % len(images)
if len(stacks2p) > 0:
self.imagestring += '2pStacks: %d ' % len(stacks2p)
if len(images2p) > 0:
self.imagestring += '2pImages: %d ' % len(images2p)
if len(videos) > 0:
self.imagestring += 'Videos: %d' % len(videos)
if len(images) + len(stacks2p) + len(images2p) + len(videos) == 0:
self.imagestring = 'No Images or Videos'
if anyprotocols:
ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + self.protocolstring + self.imagestring + ' \t'
else:
ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + '<No complete protocols> \t' + self.imagestring + ' \t'
self.outputString(ostring)
def getSummary(self):
"""
getSummary is the entry point for scanning through all the data files in a given directory,
returning information about those within the date range, with details as specified by the options
"""
allfiles = os.listdir(self.basedir)
days = []
for thisfile in allfiles:
m = self.daytype.match(thisfile)
if m == '.DS_Store':
continue
if m is None:
# print 'Top level file %s is incorrectly placed ' % thisfile
continue # no match
if len(m.groups()) >= 3: # perfect match
# print m.groups()
idl = [int(d) for d in m.groups()]
id = idl[0] * 1e4 + idl[1] * 1e2 + idl[2]
# print 'id: ', id
# print 'minday: ', minday
if self.daylist is None:
if id >= self.minday and id <= self.maxday:
days.append(thisfile) # was [0:10]
else:
#print 'using daylist, thisfile: ', thisfile[0:10]
#print 'daylist: ', daylist
if thisfile[0:10] in self.daylist:
days.append(thisfile)
if self.monitor:
print 'Days reported: ', days
for day in days:
if self.monitor:
print 'processing day: %s' % day
self.daystring = '%s \t' % (day)
dh = DataManager.getDirHandle(os.path.join(self.basedir, day),
create=False)
dx = self.dataModel.getDayInfo(dh)
if dx is not None and 'description' in dx.keys() and len(
dx['description']) > 0:
l = self.twd['day'].wrap(dx['description'])
for i in l:
i = i.replace(
'\t', ' '
) # clean out tabs so printed formatting is not confused
self.daystring += i
else:
self.daystring += ' [no description]'
self.daystring += ' \t'
if dx is not None and 'notes' in dx.keys() and len(
dx['notes']) > 0:
l = self.tw['day'].wrap(dx['notes'])
for i in l:
i = i.replace(
'\t', ' '
) # clean out tabs so printed formatting is not confused
self.daystring += i
else:
self.daystring += ' [no notes]'
self.daystring += ' \t'
self.doSlices(os.path.join(self.basedir, day))
os.closerange(8, 65535) # close all files in each iteration
gc.collect()
import os, sys
d = os.path.split(os.path.abspath(__file__))[0]
d1 = os.path.split(d)[0]
d2 = os.path.split(d1)[0]
sys.path.extend([d1, d2])
from acq4.util.DirTreeWidget import *
from DirTreeLoader import *
from acq4.util.DataManager import *
from PyQt4 import QtCore,QtGui
app = QtGui.QApplication([])
dm = DataManager()
dh = dm.getDirHandle(d)['testDir']
class Loader(DirTreeLoader):
def new(self):
print "NEW"
return True
def save(self, fh):
open(fh.name(), 'w').write("SAVED")
print "SAVED"
return True
def load(self, fh):
print "LOADED:", open(fh.name()).read()
return True
def dh(self):
if self._dh is None:
self._dh = DataManager.getDirHandle(self._filepath)
return self._dh
