def test_datamanager(): rh = dm.getDirHandle(root) # check handles are cached rh2 = dm.getDirHandle(root) assert rh is rh2 # test meta info is stored and reloaded correctly rh.setInfo({'test_int': 1, 'test_str': 'xxx'}) assert rh.info()['test_int'] == 1 assert rh.info()['test_str'] == 'xxx' # Create a subdir with meta info d1 = rh.mkdir('subdir', info={'a': 'b'}) assert d1.info()['a'] == 'b' assert d1.shortName() == 'subdir' assert d1.name() == os.path.join(rh.name(), 'subdir') # Create a DirTreeWidget; check that the contents are correct. dw = DirTreeWidget(baseDirHandle=rh) assert dw.topLevelItemCount() == 1 item = dw.topLevelItem(0) assert item.text(0) == d1.shortName() assert item.handle is d1 assert item.childCount() == 0 # Create a subdir and update the tree widget d2 = d1.mkdir('subdir2') dw.rebuildChildren(item) assert item.childCount() == 1 # test _getTree d3 = rh.mkdir('subdir3') assert d3.name() not in dm.dm._getTree(d1.name()) assert d2.name() in dm.dm._getTree(d1.name()) # # root # + subdir # | + subdir2 # + subdir3 # assert d1.name(relativeTo=rh) == 'subdir' assert d2.name(relativeTo=rh) == os.path.join('subdir', 'subdir2') assert d2.name(relativeTo=d1) == 'subdir2' assert d2.name(relativeTo=d2) == '' assert d1.name(relativeTo=d2) == '..' assert rh.name(relativeTo=d2) == os.path.join('..', '..') assert d3.name(relativeTo=d2) == os.path.join('..', '..', 'subdir3') assert d2.name(relativeTo=d3) == os.path.join('..', 'subdir', 'subdir2') # rename subdir from tree widget item.setText(0, 'subdir_renamed') assert d1.shortName() == 'subdir_renamed' # delete subdir d1.delete() dw.rebuildTree() assert dw.topLevelItemCount() == 1
def __init__(self, baseDir, sortMode='alpha', create=False, *args): QtGui.QWidget.__init__(self, *args) self.ui = Ui_Form() self.ui.setupUi(self) if isinstance(baseDir, basestring): baseDir = DataManager.getDirHandle(baseDir, create=create) self.baseDir = baseDir self.currentFile = None #self.fileTree = DirTreeModel(baseDir) #self.ui.fileTree.setModel(self.fileTree) self.ui.fileTree.setSortMode(sortMode) self.ui.fileTree.setBaseDirHandle(baseDir) self.deleteState = 0 self.ui.deleteBtn.focusOutEvent = self.delBtnLostFocus self.ui.newBtn.clicked.connect(self.newClicked) self.ui.newDirBtn.clicked.connect(self.newDirClicked) self.ui.saveBtn.clicked.connect(self.saveClicked) self.ui.loadBtn.clicked.connect(self.loadClicked) self.ui.saveAsBtn.clicked.connect(self.saveAsClicked) self.ui.deleteBtn.clicked.connect(self.deleteClicked) self.ui.fileTree.itemDoubleClicked.connect(self.loadClicked)
def doCells(self, slice): """ process all of the cells from a slice :param slice: :return nothing: """ allfiles = os.listdir(slice) celltype = re.compile("(cell_)(\d{3,3})") cells = [] for thisfile in allfiles: m = celltype.match(thisfile) if m is None: continue if len(m.groups()) == 2: cells.append(thisfile) for cell in cells: self.cellstring = '%s\t' % (cell) dh = DataManager.getDirHandle(os.path.join(slice, cell), create=False) cl = self.dataModel.getCellInfo(dh) if cl is not None and 'notes' in cl.keys() and len( cl['notes']) > 0: l = self.tw['cell'].wrap(cl['notes']) for i in l: i = i.replace( '\t', ' ' ) # clean out tabs so printed formatting is not confused self.cellstring += i else: self.cellstring += ' No cell notes' self.cellstring += ' \t' self.cell_summary(dh) # if cl is not None and 'description' in cl.keys() and len(cl['description']) > 0: # l = self.twd['cell'].wrap(cl['description']) # for i in l: # self.cellstring += i # else: # self.cellstring += ' No cell description' self.cellstring += ' \t' self.doProtocols(os.path.join(slice, cell)) DataManager.cleanup() # clean up after each cell del dh gc.collect()
def doSlices(self, day): """ process all of the slices for a given day :param day: :return nothing: """ allfiles = os.listdir(day) slicetype = re.compile("(slice\_)(\d{3,3})") slices = [] for thisfile in allfiles: m = slicetype.match(thisfile) if m is None: continue if len(m.groups()) == 2: slices.append(thisfile) for slice in slices: self.slicestring = '%s \t' % (slice) dh = DataManager.getDirHandle(os.path.join(day, slice), create=False) sl = self.dataModel.getSliceInfo(dh) # if sl is not None and 'description' in sl.keys() and len(sl['description']) > 0: # l = self.twd['slice'].wrap(sl['description']) # for i in l: # self.slicestring += i # else: # self.slicestring += ' No slice description' # self.slicestring += '\t' if sl is not None and 'notes' in sl.keys() and len( sl['notes']) > 0: l = self.tw['slice'].wrap(sl['notes']) for i in l: i = i.replace( '\t', ' ' ) # clean out tabs so printed formatting is not confused self.slicestring += i else: self.slicestring += ' No slice notes' self.slicestring += ' \t' self.doCells(os.path.join(day, slice)) DataManager.cleanup() del dh gc.collect()
def saveStateFile(self, filename): dh = DataManager.getDirHandle(os.path.dirname(filename)) state = self.saveState(relativeTo=dh) json.dump(state, open(filename, 'w'), indent=4, cls=Encoder)
import os, sys d = os.path.split(os.path.abspath(__file__))[0] d1 = os.path.split(d)[0] d2 = os.path.split(d1)[0] sys.path.extend([d1, d2]) from acq4.util.DirTreeWidget import * from DirTreeLoader import * from acq4.util.DataManager import * from PyQt4 import QtCore, QtGui app = QtGui.QApplication([]) dm = DataManager() dh = dm.getDirHandle(d)['testDir'] class Loader(DirTreeLoader): def new(self): print "NEW" return True def save(self, fh): open(fh.name(), 'w').write("SAVED") print "SAVED" return True def load(self, fh): print "LOADED:", open(fh.name()).read() return True
def doProtocols(self, cell): """ process all of the protocols for a given cell :param cell: :return nothing: """ allfiles = os.listdir(cell) #celltype = re.compile("(Cell_)(\d{3,3})") protocols = [] nonprotocols = [] anyprotocols = False images = [] # tiff stacks2p = [] images2p = [] videos = [] endmatch = re.compile( "[\_(\d{3,3})]$") # look for _lmn at end of directory name for thisfile in allfiles: if os.path.isdir(os.path.join(cell, thisfile)): protocols.append(thisfile) else: nonprotocols.append(thisfile) self.protocolstring = '' if self.InvestigateProtocols is True: self.summarystring = 'NaN \t' * 6 for np, protocol in enumerate(protocols): dh = DataManager.getDirHandle(os.path.join(cell, protocol), create=False) if np == 0: self.cell_summary(dh) if self.monitor: print 'Investigating Protocol: %s', dh.name() dirs = dh.subDirs() protocolok = True # assume that protocol is ok modes = [] nexpected = len( dirs ) # acq4 writes dirs before, so this is the expected fill ncomplete = 0 # count number actually done clampDevices = self.dataModel.getClampDeviceNames(dh) # must handle multiple data formats, even in one experiment... if clampDevices is not None: data_mode = dh.info()['devices'][clampDevices[0]][ 'mode'] # get mode from top of protocol information else: # try to set a data mode indirectly if 'devices' not in dh.info().keys(): protocolok = False # can't parse protocol device... continue devices = dh.info()['devices'].keys( ) # try to get clamp devices from another location #print dir(self.dataModel) for kc in self.dataModel.knownClampNames(): if kc in devices: clampDevices = [kc] try: data_mode = dh.info()['devices'][ clampDevices[0]]['mode'] except: data_mode = 'Unknown' # protocolok = False # print '<<cannot read protocol data mode>>' if data_mode not in modes: modes.append(data_mode) for i, directory_name in enumerate( dirs ): # dirs has the names of the runs within the protocol data_dir_handle = dh[ directory_name] # get the directory within the protocol try: data_file_handle = self.dataModel.getClampFile( data_dir_handle) # get pointer to clamp data except: data_file_handle = None if data_file_handle is not None: # no clamp file found - skip ncomplete += 1 # Check if there is no clamp file for this iteration of the protocol # Usually this indicates that the protocol was stopped early. # data_file = data_file_handle.read() try: self.holding = self.dataModel.getClampHoldingLevel( data_file_handle) except: self.holding = 0. try: self.amp_settings = self.dataModel.getWCCompSettings( data_file_handle) except: self.amp_settings = None #raise ValueError('complete = %d when failed' % ncomplete) #else: # break # do not keep looking if the file is not found DataManager.cleanup() # close all opened files # del dh gc.collect( ) # and force garbage collection of freed objects inside the loop if modes == []: modes = ['Unknown mode'] if protocolok and ncomplete == nexpected: # accumulate protocols self.protocolstring += '[{:<s}: {:s} {:d}], '.format( protocol, modes[0][0], ncomplete) anyprotocols = True # indicate that ANY protocol ran to completion else: if self.reportIncompleteProtocols: self.protocolstring += '[{:<s}, ({:s}, {:d}/{:d}, Incomplete)], '.format( protocol, modes[0][0], ncomplete, nexpected) DataManager.cleanup() del dh gc.collect() else: self.protocolstring += 'Protocols: ' anyprotocols = True prots = {} for protocol in protocols: m = endmatch.search(protocol) if m is not None: p = protocol[:-4] else: p = protocol if p not in prots.keys(): prots[p] = 1 else: prots[p] += 1 if len(prots.keys()) > 0: self.protocolstring += '[' for p in prots.keys(): self.protocolstring += '{:<s}({:<d}), '.format(p, prots[p]) self.protocolstring += ']' else: self.protocolstring = '<No protocols found>' self.protocolstring += ' \t' for thisfile in nonprotocols: # if os.path.isdir(os.path.join(cell, thisfile)): # skip protocols # continue x = self.img_re.match(thisfile) # look for image files if x is not None: images.append(thisfile) x = self.s2p_re.match(thisfile) # two photon stacks if x is not None: stacks2p.append(thisfile) x = self.i2p_re.match(thisfile) # simple two photon images if x is not None: images2p.append(thisfile) x = self.video_re.match(thisfile) # video images if x is not None: videos.append(thisfile) self.imagestring = '' if len(images) > 0: self.imagestring += 'Images: %d ' % len(images) if len(stacks2p) > 0: self.imagestring += '2pStacks: %d ' % len(stacks2p) if len(images2p) > 0: self.imagestring += '2pImages: %d ' % len(images2p) if len(videos) > 0: self.imagestring += 'Videos: %d' % len(videos) if len(images) + len(stacks2p) + len(images2p) + len(videos) == 0: self.imagestring = 'No Images or Videos' if anyprotocols: ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + self.protocolstring + self.imagestring + ' \t' else: ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + '<No complete protocols> \t' + self.imagestring + ' \t' self.outputString(ostring)
def getSummary(self): """ getSummary is the entry point for scanning through all the data files in a given directory, returning information about those within the date range, with details as specified by the options """ allfiles = os.listdir(self.basedir) days = [] for thisfile in allfiles: m = self.daytype.match(thisfile) if m == '.DS_Store': continue if m is None: # print 'Top level file %s is incorrectly placed ' % thisfile continue # no match if len(m.groups()) >= 3: # perfect match # print m.groups() idl = [int(d) for d in m.groups()] id = idl[0] * 1e4 + idl[1] * 1e2 + idl[2] # print 'id: ', id # print 'minday: ', minday if self.daylist is None: if id >= self.minday and id <= self.maxday: days.append(thisfile) # was [0:10] else: #print 'using daylist, thisfile: ', thisfile[0:10] #print 'daylist: ', daylist if thisfile[0:10] in self.daylist: days.append(thisfile) if self.monitor: print 'Days reported: ', days for day in days: if self.monitor: print 'processing day: %s' % day self.daystring = '%s \t' % (day) dh = DataManager.getDirHandle(os.path.join(self.basedir, day), create=False) dx = self.dataModel.getDayInfo(dh) if dx is not None and 'description' in dx.keys() and len( dx['description']) > 0: l = self.twd['day'].wrap(dx['description']) for i in l: i = i.replace( '\t', ' ' ) # clean out tabs so printed formatting is not confused self.daystring += i else: self.daystring += ' [no description]' self.daystring += ' \t' if dx is not None and 'notes' in dx.keys() and len( dx['notes']) > 0: l = self.tw['day'].wrap(dx['notes']) for i in l: i = i.replace( '\t', ' ' ) # clean out tabs so printed formatting is not confused self.daystring += i else: self.daystring += ' [no notes]' self.daystring += ' \t' self.doSlices(os.path.join(self.basedir, day)) os.closerange(8, 65535) # close all files in each iteration gc.collect()
import os, sys d = os.path.split(os.path.abspath(__file__))[0] d1 = os.path.split(d)[0] d2 = os.path.split(d1)[0] sys.path.extend([d1, d2]) from acq4.util.DirTreeWidget import * from DirTreeLoader import * from acq4.util.DataManager import * from PyQt4 import QtCore,QtGui app = QtGui.QApplication([]) dm = DataManager() dh = dm.getDirHandle(d)['testDir'] class Loader(DirTreeLoader): def new(self): print "NEW" return True def save(self, fh): open(fh.name(), 'w').write("SAVED") print "SAVED" return True def load(self, fh): print "LOADED:", open(fh.name()).read() return True
def dh(self): if self._dh is None: self._dh = DataManager.getDirHandle(self._filepath) return self._dh