示例#1
0
def test_datamanager():

    rh = dm.getDirHandle(root)

    # check handles are cached
    rh2 = dm.getDirHandle(root)
    assert rh is rh2

    # test meta info is stored and reloaded correctly
    rh.setInfo({'test_int': 1, 'test_str': 'xxx'})
    assert rh.info()['test_int'] == 1
    assert rh.info()['test_str'] == 'xxx'

    # Create a subdir with meta info
    d1 = rh.mkdir('subdir', info={'a': 'b'})
    assert d1.info()['a'] == 'b'
    assert d1.shortName() == 'subdir'
    assert d1.name() == os.path.join(rh.name(), 'subdir')

    # Create a DirTreeWidget; check that the contents are correct.
    dw = DirTreeWidget(baseDirHandle=rh)
    assert dw.topLevelItemCount() == 1
    item = dw.topLevelItem(0)
    assert item.text(0) == d1.shortName()
    assert item.handle is d1
    assert item.childCount() == 0

    # Create a subdir and update the tree widget
    d2 = d1.mkdir('subdir2')
    dw.rebuildChildren(item)
    assert item.childCount() == 1

    # test _getTree
    d3 = rh.mkdir('subdir3')
    assert d3.name() not in dm.dm._getTree(d1.name())
    assert d2.name() in dm.dm._getTree(d1.name())

    #
    # root
    #   + subdir
    #   |   + subdir2
    #   + subdir3
    #
    assert d1.name(relativeTo=rh) == 'subdir'
    assert d2.name(relativeTo=rh) == os.path.join('subdir', 'subdir2')
    assert d2.name(relativeTo=d1) == 'subdir2'
    assert d2.name(relativeTo=d2) == ''
    assert d1.name(relativeTo=d2) == '..'
    assert rh.name(relativeTo=d2) == os.path.join('..', '..')
    assert d3.name(relativeTo=d2) == os.path.join('..', '..', 'subdir3')
    assert d2.name(relativeTo=d3) == os.path.join('..', 'subdir', 'subdir2')

    # rename subdir from tree widget
    item.setText(0, 'subdir_renamed')
    assert d1.shortName() == 'subdir_renamed'

    # delete subdir
    d1.delete()
    dw.rebuildTree()
    assert dw.topLevelItemCount() == 1
示例#2
0
    def restoreState(self, state, rootPath=None):
        if state.get('contents', None) != 'MosaicEditor_save':
            raise TypeError("This does not appear to be MosaicEditor save data.")
        if state['version'][0] > self._saveVersion[0]:
            raise TypeError("Save data has version %d.%d, but this MosaicEditor only supports up to version %d.x." % (state['version'][0], state['version'][1], self._saveVersion[0]))

        if not self.clear():
            return

        root = state['rootPath']
        if root == '':
            # data was stored with no root path; filenames should be absolute
            root = None
        else:
            # data was stored with no root path; filenames should be relative to the loaded file            
            root = DataManager.getHandle(rootPath)
            
        for itemState in state['items']:
            fname = itemState['filename']
            if root is None:
                fh = DataManager.getHandle(fh)
            else:
                fh = root[fname]
            item = self.addFile(fh, name=itemState['name'], inheritTransform=False)
            item.restoreState(itemState)
        
        self.canvas.view.setState(state['view'])
示例#3
0
def getSiteNWBs(basepath):
    """ Get all experiment NWBs in the slice*/site* folders """

    matches = []

    dirHandleBase = adm.getHandle(basepath)
    dirHandleBase.checkIndex()

    # start in basepath
    for k in dirHandleBase.ls():
        if not dirHandleBase.isManaged(k):
            continue

        slicePath = os.path.join(basepath, k)

        if os.path.isfile(slicePath):
            continue

        # base/slice
        dirHandleSlice = adm.getHandle(slicePath)
        dirHandleSlice.checkIndex()

        for k in dirHandleSlice.ls():
            if not dirHandleSlice.isManaged(k):
                continue

            sitePath = os.path.join(slicePath, k)

            if os.path.isfile(sitePath):
                continue

            # base/slice/site
            all_nwbs = glob.glob(os.path.join(sitePath, '*.nwb'))
            all_pxps = glob.glob(os.path.join(sitePath, '*.pxp'))

            if len(all_nwbs) != 1:
                print "The site folder \"%s\" will be ignored as it holds more than one NWB file." % sitePath
                print all_nwbs
                continue

            if len(all_pxps) != 1:
                print "The site folder \"%s\" will be ignored as it holds more than one PXP file." % sitePath
                print all_pxps
                continue

            siteNWB = all_nwbs[0]
            sitePXP = all_pxps[0]

            if not os.path.isfile(siteNWB):
                print "The site folder \"%s\" will be ignored as it is missing the mandatory NWB file." % sitePath
                continue

            if not os.path.isfile(sitePXP):
                print "The site folder \"%s\" will be ignored as it is missing the mandatory PXP file." % sitePath
                continue

            matches.append(os.path.abspath(siteNWB))

    return matches
示例#4
0
    def validate_script(self):
        """
        validate the current script - by checking the existence of the files needed for the analysis

        :return: False if cannot find files; True if all are found
        """
        # if self.script['module'] != 'IVCurve':
        #     print 'Script is not for IVCurve (found %s)' % self.script['module']
        #     return False
        if 'directory' in self.script.keys():
            try:

                #print dir(self.dataManager())
                self.dataManager().setBaseDir(self.script['directory'])
                print('Set base dir to: {:s}'.format(self.script['directory']))
            except:
                print(
                    'ScriptProcessor:read_script: \n   Cannot set base directory to %s\n   Likely directory was not found'
                    % self.script['directory'])
                return False

        all_found = True
        trailingchars = [
            c for c in map(chr, range(97, 123))
        ]  # trailing chars used to identify different parts of a cell's data
        for c in self.script['Cells']:
            if self.script['Cells'][c]['include'] is False:
                continue
            sortedkeys = sorted(
                self.script['Cells'][c]
                ['choice'].keys())  # sort by order of recording
            for p in sortedkeys:
                pr = self.script[
                    'protocol'] + '_' + p  # add the underscore here
                if c[-1] in trailingchars:
                    cell = c[:-1]
                else:
                    cell = c
                fn = os.path.join(cell, pr)
                #print fn
                #print 'dm selected file: ', self.dataManager().selectedFile()
                if 'directory' in self.script.keys():
                    dm_selected_file = self.script['directory']
                else:
                    dm_selected_file = self.dataManager().selectedFile().name()
                DataManager.cleanup()
                gc.collect()
                fullpath = os.path.join(dm_selected_file, fn)
                file_ok = os.path.exists(fullpath)
                if file_ok:
                    print('File found: {:s}'.format(fullpath))
                else:
                    print('  current dataManager self.dm points to file: ',
                          dm_selected_file)
                    print('  and file not found was: ', fullpath)
                    all_found = False
                #else:
                #    print 'file found ok: %s' % fullpath
        return all_found
示例#5
0
    def restoreState(self, state, rootPath=None):
        if state.get('contents', None) != 'MosaicEditor_save':
            raise TypeError(
                "This does not appear to be MosaicEditor save data.")
        if state['version'][0] > self._saveVersion[0]:
            raise TypeError(
                "Save data has version %d.%d, but this MosaicEditor only supports up to version %d.x."
                % (state['version'][0], state['version'][1],
                   self._saveVersion[0]))

        if not self.clear():
            return

        root = state['rootPath']
        if root == '':
            # data was stored with no root path; filenames should be absolute
            root = None
        else:
            # data was stored with no root path; filenames should be relative to the loaded file
            root = DataManager.getHandle(rootPath)

        loadfail = []
        for itemState in state['items']:
            fname = itemState.get('filename')

            if fname is None:
                # create item from scratch and restore state
                itemtype = itemState.get('type')
                if itemtype not in items.itemTypes():
                    # warn the user later on that we could not load this item
                    loadfail.append((itemState.get('name'),
                                     'Unknown item type "%s"' % itemtype))
                    continue
                item = self.addItem(type=itemtype, name=itemState['name'])
            else:
                # normalize file name for this OS
                revsep = {'/': '\\', '\\': '/'}[os.path.sep]
                fname = fname.replace(revsep, os.path.sep)

                # create item by loading file and restore state
                if root is None:
                    fh = DataManager.getHandle(fh)
                else:
                    fh = root[fname]
                item = self.addFile(fh,
                                    name=itemState['name'],
                                    inheritTransform=False)
            item.restoreState(itemState)

        self.canvas.view.setState(state['view'])
        if len(loadfail) > 0:
            msg = "\n".join(["%s: %s" % m for m in loadfail])
            raise Exception("Failed to load some items:\n%s" % msg)
def getSiteNWBs(basepath):
    """ Get all experiment NWBs in the slice*/site* folders """

    matches = []

    dirHandleBase = adm.getHandle(basepath)
    dirHandleBase.checkIndex()

    # start in basepath
    for k in dirHandleBase.ls():
        if not dirHandleBase.isManaged(k):
            continue

        slicePath = os.path.join(basepath, k)

        if os.path.isfile(slicePath):
            continue

        # base/slice
        dirHandleSlice = adm.getHandle(slicePath)
        dirHandleSlice.checkIndex()

        for k in dirHandleSlice.ls():
            if not dirHandleSlice.isManaged(k):
                continue

            sitePath = os.path.join(slicePath, k)

            if os.path.isfile(sitePath):
                continue

            # base/slice/site
            all_nwbs = glob.glob(os.path.join(sitePath, '*.nwb'))

            if len(all_nwbs) != 1:
                print "The site folder \"%s\" will be ignored as it holds not exactly one NWB file." % sitePath
                print all_nwbs
                continue

            siteNWB = all_nwbs[0]

            if not os.path.isfile(siteNWB):
                print "The site folder \"%s\" will be ignored as it is missing the mandatory NWB file." % sitePath
                continue

            matches.append(os.path.abspath(siteNWB))

    return matches
示例#7
0
    def __init__(self, baseDir, sortMode='alpha', create=False, *args):
        QtGui.QWidget.__init__(self, *args)
        self.ui = Ui_Form()
        self.ui.setupUi(self)
        if isinstance(baseDir, basestring):
            baseDir = DataManager.getDirHandle(baseDir, create=create)
        self.baseDir = baseDir
        self.currentFile = None

        #self.fileTree = DirTreeModel(baseDir)
        #self.ui.fileTree.setModel(self.fileTree)
        self.ui.fileTree.setSortMode(sortMode)
        self.ui.fileTree.setBaseDirHandle(baseDir)

        self.deleteState = 0

        self.ui.deleteBtn.focusOutEvent = self.delBtnLostFocus

        self.ui.newBtn.clicked.connect(self.newClicked)
        self.ui.newDirBtn.clicked.connect(self.newDirClicked)
        self.ui.saveBtn.clicked.connect(self.saveClicked)
        self.ui.loadBtn.clicked.connect(self.loadClicked)
        self.ui.saveAsBtn.clicked.connect(self.saveAsClicked)
        self.ui.deleteBtn.clicked.connect(self.deleteClicked)
        self.ui.fileTree.itemDoubleClicked.connect(self.loadClicked)
示例#8
0
    def __init__(self, baseDir, sortMode='alpha', create=False, *args):
        QtGui.QWidget.__init__(self, *args)
        self.ui = Ui_Form()
        self.ui.setupUi(self)
        if isinstance(baseDir, basestring):
            baseDir = DataManager.getDirHandle(baseDir, create=create)
        self.baseDir = baseDir
        self.currentFile = None
        
        #self.fileTree = DirTreeModel(baseDir)
        #self.ui.fileTree.setModel(self.fileTree)
        self.ui.fileTree.setSortMode(sortMode)
        self.ui.fileTree.setBaseDirHandle(baseDir)
        
        self.deleteState = 0

        self.ui.deleteBtn.focusOutEvent = self.delBtnLostFocus

        self.ui.newBtn.clicked.connect(self.newClicked)
        self.ui.newDirBtn.clicked.connect(self.newDirClicked)
        self.ui.saveBtn.clicked.connect(self.saveClicked)
        self.ui.loadBtn.clicked.connect(self.loadClicked)
        self.ui.saveAsBtn.clicked.connect(self.saveAsClicked)
        self.ui.deleteBtn.clicked.connect(self.deleteClicked)
        self.ui.fileTree.itemDoubleClicked.connect(self.loadClicked)
示例#9
0
    def loadFileRequested(self, files):
        self.canvas = self.getElement('Canvas')
        if files is None:
            return

        for f in files:
            if f in self.files:   ## Do not allow loading the same file more than once
                item = self.files[f]
                item.show()  # just show the file; but do not load it
                continue
            
            if f.isFile():  # add specified files
                item = self.canvas.addFile(f)
            elif f.isDir():  # Directories are more complicated
                if self.dataModel.dirType(f) == 'Cell':  #  If it is a cell, just add the cell "Marker" to the plot
                # note: this breaks loading all images in Cell directory (need another way to do that)
                    item = self.canvas.addFile(f)
                else:  # in all other directory types, look for MetaArray files
                    filesindir = glob.glob(f.name() + '/*.ma')
                    for fd in filesindir:  # add files in the directory (ma files: e.g., images, videos)
                        try:
                            fdh = DataManager.getFileHandle(fd) # open file to get handle.
                        except IOError:
                            continue # just skip file
                        item = self.canvas.addFile(fdh)  # add it
                        self.amendFile(f, item)
                    if len(filesindir) == 0:  # add protocol sequences
                        item = self.canvas.addFile(f)
        self.canvas.selectItem(item)
        self.canvas.autoRange()
示例#10
0
    def loadFileRequested(self, files):
        if files is None:
            return

        for f in files:
            if f.shortName().endswith('.mosaic'):
                self.loadStateFile(f.name())
                continue

            if f in self.files:  ## Do not allow loading the same file more than once
                item = self.files[f]
                item.show()  # just show the file; but do not load it
                continue

            if f.isFile():  # add specified files
                item = self.addFile(f)
            elif f.isDir():  # Directories are more complicated
                if self.dataModel.dirType(
                        f
                ) == 'Cell':  #  If it is a cell, just add the cell "Marker" to the plot
                    item = self.canvas.addFile(f)
                else:  # in all other directory types, look for MetaArray files
                    filesindir = glob.glob(f.name() + '/*.ma')
                    for fd in filesindir:  # add files in the directory (ma files: e.g., images, videos)
                        try:
                            fdh = DataManager.getFileHandle(
                                fd)  # open file to get handle.
                        except IOError:
                            continue  # just skip file
                        item = self.addFile(fdh)
                    if len(filesindir) == 0:  # add protocol sequences
                        item = self.addFile(f)
        self.canvas.autoRange()
示例#11
0
    def doCells(self, slice):
        """
        process all of the cells from a slice
        :param slice:
        :return nothing:
        """
        allfiles = os.listdir(slice)
        celltype = re.compile("(cell_)(\d{3,3})")
        cells = []
        for thisfile in allfiles:
            m = celltype.match(thisfile)
            if m is None:
                continue
            if len(m.groups()) == 2:
                cells.append(thisfile)
        for cell in cells:
            self.cellstring = '%s\t' % (cell)
            dh = DataManager.getDirHandle(os.path.join(slice, cell),
                                          create=False)
            cl = self.dataModel.getCellInfo(dh)
            if cl is not None and 'notes' in cl.keys() and len(
                    cl['notes']) > 0:
                l = self.tw['cell'].wrap(cl['notes'])
                for i in l:
                    i = i.replace(
                        '\t', '    '
                    )  # clean out tabs so printed formatting is not confused
                    self.cellstring += i
            else:
                self.cellstring += ' No cell notes'
            self.cellstring += ' \t'
            self.cell_summary(dh)

            # if cl is not None and 'description' in cl.keys() and len(cl['description']) > 0:
            #     l = self.twd['cell'].wrap(cl['description'])
            #     for i in l:
            #         self.cellstring += i
            # else:
            #     self.cellstring += ' No cell description'

            self.cellstring += ' \t'
            self.doProtocols(os.path.join(slice, cell))
            DataManager.cleanup()  # clean up after each cell
            del dh
            gc.collect()
示例#12
0
    def doSlices(self, day):
        """
        process all of the slices for a given day
        :param day:
        :return nothing:
        """

        allfiles = os.listdir(day)
        slicetype = re.compile("(slice\_)(\d{3,3})")
        slices = []
        for thisfile in allfiles:
            m = slicetype.match(thisfile)
            if m is None:
                continue
            if len(m.groups()) == 2:
                slices.append(thisfile)
        for slice in slices:
            self.slicestring = '%s \t' % (slice)
            dh = DataManager.getDirHandle(os.path.join(day, slice),
                                          create=False)
            sl = self.dataModel.getSliceInfo(dh)

            # if sl is not None and 'description' in sl.keys() and len(sl['description']) > 0:
            #     l = self.twd['slice'].wrap(sl['description'])
            #     for i in l:
            #         self.slicestring += i
            # else:
            #     self.slicestring += ' No slice description'
            # self.slicestring += '\t'

            if sl is not None and 'notes' in sl.keys() and len(
                    sl['notes']) > 0:
                l = self.tw['slice'].wrap(sl['notes'])
                for i in l:
                    i = i.replace(
                        '\t', '    '
                    )  # clean out tabs so printed formatting is not confused
                    self.slicestring += i
            else:
                self.slicestring += ' No slice notes'
            self.slicestring += ' \t'
            self.doCells(os.path.join(day, slice))
            DataManager.cleanup()
            del dh
            gc.collect()
示例#13
0
def addSiteContents(siteNWBs, filesToInclude, slicePath, siteName):
    """ Add site specific entries to the NWB file """

    sitePath = os.path.join(slicePath, siteName)
    siteIndex = os.path.join(sitePath, ".index")

    if len(siteNWBs) != 1:
        print "Expected exactly one NWB file belonging to site folder %s, skipping it." % sitePath
        return 1

    sitePXP = glob.glob(os.path.join(sitePath, '*.pxp'))[0]

    addDataSource(siteNWBs, sitePXP)

    dh = adm.getHandle(sitePath)
    dh.checkIndex()

    data = encodeAsJSONString(dh["."].info())
    appendMiscFileToNWB(siteNWBs, "%s_index_meta" % siteName, data)
    appendMiscFileToNWB(siteNWBs, "%s_index" % siteName,
                        getFileContents(siteIndex))

    for k in dh.ls():

        if not dh.isManaged(k):
            continue

        path = os.path.join(sitePath, k)

        if os.path.isdir(path):
            raise NameError("Unexpected folder \"%s\" in \"%s\"." %
                            (k, sitePath))
        elif os.path.isfile(path):  # check if we need to handle it

            if fileShouldBeSkipped(path, filesToInclude):
                continue

            filedesc = encodeAsJSONString(dh[k].info())

            if path.endswith(".tif"):
                appendImageFileToNWB(siteNWBs, path, filedesc)
            elif path.endswith(".ma"):
                appendMAFile(siteNWBs, path, filedesc)
            elif path.endswith(".log"):
                appendPseudoYamlLog(siteNWBs, path, k, filedesc)
            else:
                raise NameError(
                    "Unexpected file type \"%s\" in index \"%s\"." %
                    (k, siteIndex))
        else:
            raise NameError("Unexpected key \"%s\" in index \"%s\"." %
                            (k, siteIndex))
示例#14
0
 def __init__(self, atlas, host):
     Atlas.AtlasCtrlWidget.__init__(self, atlas, host)
     
     self.atlasDir = os.path.split(os.path.abspath(__file__))[0]
     
     ## add ThalamocorticalMarker to canvas
     fh = DataManager.getHandle(os.path.join(self.atlasDir, 'images', 'ThalamocorticalMarker.svg'))
     self.canvas.addFile(fh, pos=(-0.001283, -0.000205), scale=[3.78e-6, 3.78e-6], index=0, movable=False, z=10000)        
  
     ## add CortexROI
     self.roi = CortexROI([-1e-3, 0])
     self.canvas.addGraphicsItem(self.roi, pos=(-1e-3, 1e-3), scale=[1e-3, 1e-3], name='CortexROI', movable=False)
     self.roi.sigRegionChangeFinished.connect(self.roiChanged)
示例#15
0
def buildCombinedNWBInternal(basepath, siteNWBs, filesToInclude):
    """ NOT FOR PUBLIC USE """

    # we have three types of keys in the main index file
    # ---------------------------------------------------------------------
    # '.'               | common description of the experiment | (unique)
    # '$existingFile'   | log file of the experiment           | (multiple)
    # '$existingFolder' | different slices for each experiment | (multiple)

    dh = adm.getHandle(basepath)
    dh.checkIndex()

    data = encodeAsJSONString(dh["."].info())
    appendMiscFileToNWB(siteNWBs, basename="main_index_meta", content=data)

    logfile = os.path.join(basepath, '.index')
    appendMiscFileToNWB(siteNWBs,
                        basename="main_index",
                        content=getFileContents(logfile))

    for elem in siteNWBs:
        os.remove(deriveOutputNWB(elem))

    for k in dh.ls():
        if not dh.isManaged(k):
            continue

        path = os.path.abspath(os.path.join(basepath, k))

        if os.path.isdir(path):  # slice folder
            addSliceContents(siteNWBs, filesToInclude, basepath, k)
        elif os.path.isfile(path):  # main log file

            data = encodeAsJSONString(dh[k].info())
            appendMiscFileToNWB(siteNWBs,
                                basename="main_logfile_meta",
                                content=data)
            appendMiscFileToNWB(siteNWBs,
                                basename="main_logfile",
                                content=getFileContents(path))
        else:
            raise NameError("Unexpected key \"%s\" in index \"%s\"" %
                            (k, logfile))

    combinedNWBs = []

    for elem in siteNWBs:
        combinedNWBs.append(deriveOutputNWB(elem))

    return combinedNWBs
示例#16
0
def addSliceContents(siteNWBs, filesToInclude, basepath, sliceName):
    """ Add entries to the slice specific NWB file """

    slicePath = os.path.join(basepath, sliceName)
    sliceIndex = os.path.join(slicePath, ".index")

    sliceNWBs = [elem for elem in siteNWBs if elem.startswith(slicePath)]

    if len(sliceNWBs) == 0:
        #print "No NWB files belong to slice folder %s, skipping it." % slicePath
        return 1

    dh = adm.getHandle(slicePath)
    dh.checkIndex()

    data = encodeAsJSONString(dh["."].info())
    appendMiscFileToNWB(siteNWBs, "%s_index_meta" % sliceName, data)
    appendMiscFileToNWB(siteNWBs, "%s_index" % sliceName,
                        getFileContents(sliceIndex))

    for k in dh.ls():
        if not dh.isManaged(k):
            continue

        path = os.path.abspath(os.path.join(slicePath, k))

        filedesc = encodeAsJSONString(dh[k].info())

        if os.path.isdir(path):  # site folder
            appendMiscFileToNWB(sliceNWBs, "%s_%s" % (sliceName, k), filedesc)
            addSiteContents(sliceNWBs, filesToInclude, slicePath, k)
        elif os.path.isfile(path):  # check if we need to handle it

            if fileShouldBeSkipped(path, filesToInclude):
                continue

            if path.endswith(".tif"):
                appendImageFileToNWB(sliceNWBs, path, filedesc)
            elif path.endswith(".ma"):
                appendMAFile(sliceNWBs, path, filedesc)
            else:
                raise NameError("Unexpected file type \"%s\" in index \"%s\"" %
                                (k, sliceIndex))
        else:
            raise NameError("Unexpected key \"%s\" in index \"%s\"" %
                            (k, sliceIndex))
def addSiteContents(siteNWBs, filesToInclude, slicePath, siteName):
    """ Add site specific entries to the NWB file """

    sitePath  = os.path.join(slicePath, siteName)
    siteIndex = os.path.join(sitePath, ".index")

    if len(siteNWBs) != 1:
        print "Expected exactly one NWB file belonging to site folder %s, skipping it." % sitePath
        return 1

    addDataSource(siteNWBs)

    dh = adm.getHandle(sitePath)
    dh.checkIndex()

    data = encodeAsJSONString(dh["."].info())
    appendMiscFileToNWB(siteNWBs, "%s_index_meta" % siteName, data)
    appendMiscFileToNWB(siteNWBs, "%s_index" % siteName, getFileContents(siteIndex))

    for k in dh.ls():

        if not dh.isManaged(k):
            continue

        path = os.path.join(sitePath, k)

        if os.path.isdir(path):
            raise NameError("Unexpected folder \"%s\" in \"%s\"." % (k, sitePath))
        elif os.path.isfile(path): # check if we need to handle it

            if fileShouldBeSkipped(path, filesToInclude):
                continue

            filedesc = encodeAsJSONString(dh[k].info())

            if path.endswith(".tif"):
                appendImageFileToNWB(siteNWBs, path, filedesc)
            elif path.endswith(".ma"):
                appendMAFile(siteNWBs, path, filedesc)
            elif path.endswith(".log"):
                appendPseudoYamlLog(siteNWBs, path, k, filedesc)
            else:
                raise NameError("Unexpected file type \"%s\" in index \"%s\"." % (k, siteIndex))
        else:
            raise NameError("Unexpected key \"%s\" in index \"%s\"." % (k, siteIndex))
示例#18
0
 def uiChanged(self):
     for item in self.images:
         self.canvas.removeItem(item)
     self.images = []
     
     state = self.stateGroup.state()
     slice = state['slicePlaneCombo']
     hem = state['hemisphereCombo']
     #flip = state['flipCheck']
     
     imgs = self.slicePlanes[slice]
     atlasDir = os.path.split(os.path.abspath(__file__))[0]
     
     for imgFile, scale, pos in imgs:
         fh = DataManager.getHandle(os.path.join(atlasDir, 'images', imgFile))
         item = self.canvas.addFile(fh, pos=pos, scale=[scale,scale], index=0, movable=False, z=10000)
         #item = self.canvas.addFile(fh, index=0, movable=False)
         self.images.append(item)
    def load_stimulus(self, recording):
        """Return an instance of stimuli.Stimulus"""

        #### I don't know whether I should try to parse this from metadata, or just find square pulses in the command waveform.
        ### I think finding square pulses would be simpler, but makes the assumption that pulses are square. Which is probably usually true.
        ### what if I check the wavegenerator widget data for the function name (pulse) and then findSquarepulses, or raise an exception if it's a different function?
        if isinstance(recording, PatchClampRecording):
            items = []

            fh = DataManager.getFileHandle(recording.meta['file_name'])
            seqDir = PatchEPhys.getParent(fh, 'ProtocolSequence')
            if seqDir is not None:
                dev_info = seqDir.info()['devices'][recording.device_id]

                if dev_info['mode'].lower() == 'vc':
                    units = 'V'
                elif dev_info['mode'].lower() == 'ic':
                    units = 'A'
                else:
                    units = None

                if dev_info['holdingCheck']:
                    items.append(stimuli.Offset(dev_info['holdingSpin']))

                stim_pulses = PatchEPhys.getStimParams(fh)

                for p in stim_pulses:
                    if p['function_type'] == 'pulse':
                        items.append(
                            stimuli.SquarePulse(p['start'], p['length'],
                                                p['amplitude']))
                    elif p['function_type'] == 'pulseTrain':
                        items.append(
                            stimuli.SquarePulseTrain(p['start'],
                                                     p['pulse_number'],
                                                     p['length'],
                                                     p['amplitude'],
                                                     p['period']))

                desc = seqDir.shortName()[:-4]
                return stimuli.Stimulus(desc, items=items, units=units)

        else:
            raise Exception('not implemented yet')
def addSliceContents(siteNWBs, filesToInclude, basepath, sliceName):
    """ Add entries to the slice specific NWB file """

    slicePath  = os.path.join(basepath, sliceName)
    sliceIndex = os.path.join(slicePath, ".index")

    sliceNWBs = [elem for elem in siteNWBs if elem.startswith(slicePath)]

    if len(sliceNWBs) == 0:
        #print "No NWB files belong to slice folder %s, skipping it." % slicePath
        return 1

    dh = adm.getHandle(slicePath)
    dh.checkIndex()

    data = encodeAsJSONString(dh["."].info())
    appendMiscFileToNWB(siteNWBs, "%s_index_meta" % sliceName, data)
    appendMiscFileToNWB(siteNWBs, "%s_index" % sliceName, getFileContents(sliceIndex))

    for k in dh.ls():
        if not dh.isManaged(k):
            continue

        path = os.path.abspath(os.path.join(slicePath, k))

        filedesc = encodeAsJSONString(dh[k].info())

        if os.path.isdir(path): # site folder
            appendMiscFileToNWB(sliceNWBs, "%s_%s" % (sliceName, k), filedesc)
            addSiteContents(sliceNWBs, filesToInclude, slicePath, k)
        elif os.path.isfile(path): # check if we need to handle it

            if fileShouldBeSkipped(path, filesToInclude):
                continue

            if path.endswith(".tif"):
                appendImageFileToNWB(sliceNWBs, path, filedesc)
            elif path.endswith(".ma"):
                appendMAFile(sliceNWBs, path, filedesc)
            else:
                raise NameError("Unexpected file type \"%s\" in index \"%s\"" % (k, sliceIndex))
        else:
            raise NameError("Unexpected key \"%s\" in index \"%s\"" % (k, sliceIndex))
def buildCombinedNWBInternal(basepath, siteNWBs, filesToInclude):
    """ NOT FOR PUBLIC USE """

    # we have three types of keys in the main index file
    # ---------------------------------------------------------------------
    # '.'               | common description of the experiment | (unique)
    # '$existingFile'   | log file of the experiment           | (multiple)
    # '$existingFolder' | different slices for each experiment | (multiple)

    dh = adm.getHandle(basepath)
    dh.checkIndex()

    data = encodeAsJSONString(dh["."].info())
    appendMiscFileToNWB(siteNWBs, basename = "main_index_meta", content = data)

    logfile = os.path.join(basepath, '.index')
    appendMiscFileToNWB(siteNWBs, basename = "main_index", content = getFileContents(logfile))

    for elem in siteNWBs:
        os.remove(deriveOutputNWB(elem))

    for k in dh.ls():
        if not dh.isManaged(k):
            continue

        path = os.path.abspath(os.path.join(basepath, k))

        if os.path.isdir(path): # slice folder
            addSliceContents(siteNWBs, filesToInclude, basepath, k)
        elif os.path.isfile(path): # main log file

            data = encodeAsJSONString(dh[k].info())
            appendMiscFileToNWB(siteNWBs, basename = "main_logfile_meta", content = data)
            appendMiscFileToNWB(siteNWBs, basename = "main_logfile", content = getFileContents(path))
        else:
            raise NameError("Unexpected key \"%s\" in index \"%s\"" % (k, logfile))

    combinedNWBs = []

    for elem in siteNWBs:
        combinedNWBs.append(deriveOutputNWB(elem))

    return combinedNWBs
示例#22
0
 def baseDir(self):
     """Return a dirHandle for the base directory used for all file names in the database."""
     if self._baseDir is None:
         dirName = self.ctrlParam('BaseDirectory')
         self._baseDir = DataManager.getHandle(dirName)
     return self._baseDir
示例#23
0
 def saveStateFile(self, filename):
     dh = DataManager.getDirHandle(os.path.dirname(filename))
     state = self.saveState(relativeTo=dh)
     json.dump(state, open(filename, 'w'), indent=4, cls=Encoder)
 def dh(self):
     if self._dh is None:
         self._dh = DataManager.getDirHandle(self._filepath)
     return self._dh
示例#25
0
文件: test.py 项目: travis-open/acq4
import os, sys

d = os.path.split(os.path.abspath(__file__))[0]
d1 = os.path.split(d)[0]
d2 = os.path.split(d1)[0]
sys.path.extend([d1, d2])

from acq4.util.DirTreeWidget import *
from DirTreeLoader import *
from acq4.util.DataManager import *
from PyQt4 import QtCore, QtGui

app = QtGui.QApplication([])

dm = DataManager()
dh = dm.getDirHandle(d)['testDir']


class Loader(DirTreeLoader):
    def new(self):
        print "NEW"
        return True

    def save(self, fh):
        open(fh.name(), 'w').write("SAVED")
        print "SAVED"
        return True

    def load(self, fh):
        print "LOADED:", open(fh.name()).read()
示例#26
0
    def doProtocols(self, cell):
        """
        process all of the protocols for a given cell
        :param cell:
        :return nothing:
        """
        allfiles = os.listdir(cell)
        #celltype = re.compile("(Cell_)(\d{3,3})")
        protocols = []
        nonprotocols = []
        anyprotocols = False
        images = []  # tiff
        stacks2p = []
        images2p = []
        videos = []
        endmatch = re.compile(
            "[\_(\d{3,3})]$")  # look for _lmn at end of directory name
        for thisfile in allfiles:
            if os.path.isdir(os.path.join(cell, thisfile)):
                protocols.append(thisfile)
            else:
                nonprotocols.append(thisfile)

        self.protocolstring = ''
        if self.InvestigateProtocols is True:
            self.summarystring = 'NaN \t' * 6
            for np, protocol in enumerate(protocols):
                dh = DataManager.getDirHandle(os.path.join(cell, protocol),
                                              create=False)
                if np == 0:
                    self.cell_summary(dh)
                if self.monitor:
                    print 'Investigating Protocol: %s', dh.name()
                dirs = dh.subDirs()
                protocolok = True  # assume that protocol is ok
                modes = []
                nexpected = len(
                    dirs
                )  # acq4 writes dirs before, so this is the expected fill
                ncomplete = 0  # count number actually done
                clampDevices = self.dataModel.getClampDeviceNames(dh)
                # must handle multiple data formats, even in one experiment...
                if clampDevices is not None:
                    data_mode = dh.info()['devices'][clampDevices[0]][
                        'mode']  # get mode from top of protocol information
                else:  # try to set a data mode indirectly
                    if 'devices' not in dh.info().keys():
                        protocolok = False  # can't parse protocol device...
                        continue
                    devices = dh.info()['devices'].keys(
                    )  # try to get clamp devices from another location
                    #print dir(self.dataModel)
                    for kc in self.dataModel.knownClampNames():
                        if kc in devices:
                            clampDevices = [kc]
                    try:
                        data_mode = dh.info()['devices'][
                            clampDevices[0]]['mode']
                    except:
                        data_mode = 'Unknown'
                        # protocolok = False
                        # print '<<cannot read protocol data mode>>'
                if data_mode not in modes:
                    modes.append(data_mode)
                for i, directory_name in enumerate(
                        dirs
                ):  # dirs has the names of the runs within the protocol
                    data_dir_handle = dh[
                        directory_name]  # get the directory within the protocol
                    try:
                        data_file_handle = self.dataModel.getClampFile(
                            data_dir_handle)  # get pointer to clamp data
                    except:
                        data_file_handle = None
                    if data_file_handle is not None:  # no clamp file found - skip
                        ncomplete += 1
                        # Check if there is no clamp file for this iteration of the protocol
                        # Usually this indicates that the protocol was stopped early.
                        # data_file = data_file_handle.read()
                        try:
                            self.holding = self.dataModel.getClampHoldingLevel(
                                data_file_handle)
                        except:
                            self.holding = 0.
                        try:
                            self.amp_settings = self.dataModel.getWCCompSettings(
                                data_file_handle)
                        except:
                            self.amp_settings = None
                            #raise ValueError('complete = %d when failed' % ncomplete)
                    #else:
                    #    break  # do not keep looking if the file is not found
                    DataManager.cleanup()  # close all opened files
                    # del dh
                    gc.collect(
                    )  # and force garbage collection of freed objects inside the loop
                if modes == []:
                    modes = ['Unknown mode']
                if protocolok and ncomplete == nexpected:  # accumulate protocols
                    self.protocolstring += '[{:<s}: {:s} {:d}], '.format(
                        protocol, modes[0][0], ncomplete)
                    anyprotocols = True  # indicate that ANY protocol ran to completion
                else:
                    if self.reportIncompleteProtocols:
                        self.protocolstring += '[{:<s}, ({:s}, {:d}/{:d}, Incomplete)], '.format(
                            protocol, modes[0][0], ncomplete, nexpected)

                DataManager.cleanup()
                del dh
                gc.collect()
        else:
            self.protocolstring += 'Protocols: '
            anyprotocols = True
            prots = {}
            for protocol in protocols:
                m = endmatch.search(protocol)
                if m is not None:
                    p = protocol[:-4]
                else:
                    p = protocol
                if p not in prots.keys():
                    prots[p] = 1
                else:
                    prots[p] += 1
            if len(prots.keys()) > 0:
                self.protocolstring += '['
                for p in prots.keys():
                    self.protocolstring += '{:<s}({:<d}), '.format(p, prots[p])
                self.protocolstring += ']'
            else:
                self.protocolstring = '<No protocols found>'
        self.protocolstring += ' \t'

        for thisfile in nonprotocols:
            #            if os.path.isdir(os.path.join(cell, thisfile)):  # skip protocols
            #                continue
            x = self.img_re.match(thisfile)  # look for image files
            if x is not None:
                images.append(thisfile)
            x = self.s2p_re.match(thisfile)  # two photon stacks
            if x is not None:
                stacks2p.append(thisfile)
            x = self.i2p_re.match(thisfile)  # simple two photon images
            if x is not None:
                images2p.append(thisfile)
            x = self.video_re.match(thisfile)  # video images
            if x is not None:
                videos.append(thisfile)
        self.imagestring = ''
        if len(images) > 0:
            self.imagestring += 'Images: %d ' % len(images)
        if len(stacks2p) > 0:
            self.imagestring += '2pStacks: %d ' % len(stacks2p)
        if len(images2p) > 0:
            self.imagestring += '2pImages: %d ' % len(images2p)
        if len(videos) > 0:
            self.imagestring += 'Videos: %d' % len(videos)
        if len(images) + len(stacks2p) + len(images2p) + len(videos) == 0:
            self.imagestring = 'No Images or Videos'

        if anyprotocols:
            ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + self.protocolstring + self.imagestring + ' \t'
        else:
            ostring = self.daystring + self.summarystring + self.slicestring + self.cellstring + '<No complete protocols> \t' + self.imagestring + ' \t'
        self.outputString(ostring)
示例#27
0
 def baseDir(self):
     """Return a dirHandle for the base directory used for all file names in the database."""
     if self._baseDir is None:
         dirName = self.ctrlParam('BaseDirectory')
         self._baseDir = DataManager.getHandle(dirName)
     return self._baseDir
示例#28
0
    def getSummary(self):
        """
        getSummary is the entry point for scanning through all the data files in a given directory,
        returning information about those within the date range, with details as specified by the options
        """
        allfiles = os.listdir(self.basedir)

        days = []
        for thisfile in allfiles:
            m = self.daytype.match(thisfile)
            if m == '.DS_Store':
                continue
            if m is None:
                # print 'Top level file %s is incorrectly placed ' % thisfile
                continue  # no match
            if len(m.groups()) >= 3:  # perfect match
                # print m.groups()
                idl = [int(d) for d in m.groups()]
                id = idl[0] * 1e4 + idl[1] * 1e2 + idl[2]
                # print 'id: ', id
                # print 'minday: ', minday
                if self.daylist is None:
                    if id >= self.minday and id <= self.maxday:
                        days.append(thisfile)  # was [0:10]
                else:
                    #print 'using daylist, thisfile: ', thisfile[0:10]
                    #print 'daylist: ', daylist
                    if thisfile[0:10] in self.daylist:
                        days.append(thisfile)
        if self.monitor:
            print 'Days reported: ', days
        for day in days:
            if self.monitor:
                print 'processing day: %s' % day
            self.daystring = '%s \t' % (day)
            dh = DataManager.getDirHandle(os.path.join(self.basedir, day),
                                          create=False)
            dx = self.dataModel.getDayInfo(dh)
            if dx is not None and 'description' in dx.keys() and len(
                    dx['description']) > 0:
                l = self.twd['day'].wrap(dx['description'])
                for i in l:
                    i = i.replace(
                        '\t', '    '
                    )  # clean out tabs so printed formatting is not confused
                    self.daystring += i
            else:
                self.daystring += ' [no description]'
            self.daystring += ' \t'
            if dx is not None and 'notes' in dx.keys() and len(
                    dx['notes']) > 0:
                l = self.tw['day'].wrap(dx['notes'])
                for i in l:
                    i = i.replace(
                        '\t', '    '
                    )  # clean out tabs so printed formatting is not confused
                    self.daystring += i
            else:
                self.daystring += ' [no notes]'
            self.daystring += ' \t'
            self.doSlices(os.path.join(self.basedir, day))
            os.closerange(8, 65535)  # close all files in each iteration
            gc.collect()
示例#29
0
文件: test.py 项目: ablot/acq4
# -*- coding: utf-8 -*-

import os, sys
d = os.path.split(os.path.abspath(__file__))[0]
d1 = os.path.split(d)[0]
d2 = os.path.split(d1)[0]
sys.path.extend([d1, d2])

from acq4.util.DirTreeWidget import *
from DirTreeLoader import *
from acq4.util.DataManager import *
from PyQt4 import QtCore,QtGui

app = QtGui.QApplication([])

dm = DataManager()
dh = dm.getDirHandle(d)['testDir']


class Loader(DirTreeLoader):
    def new(self):
        print "NEW"
        return True
        
    def save(self, fh):
        open(fh.name(), 'w').write("SAVED")
        print "SAVED"
        return True
    
    def load(self, fh):
        print "LOADED:", open(fh.name()).read()
示例#30
0
    def run_script(self):
        """
        Run a script, doing all of the requested analysis
        :return:
        """
        if self.script['testfiles']:
            return
        # settext = self.scripts_form.PSPReversal_ScriptResults_text.setPlainText
        # apptext = self.scripts_form.PSPReversal_ScriptResults_text.appendPlainText
        self.textout = ('\nScript File: {:<32s}\n'.format(self.script_name))
        # settext(self.textout)
        script_header = True  # reset the table to a print new header for each cell
        trailingchars = [
            c for c in map(chr, range(97, 123))
        ]  # trailing chars used to identify different parts of a cell's data
        self.dataManager().setBaseDir(self.script['directory'])
        ordered = sorted(self.script['Cells'].keys()
                         )  # order the analysis by date/slice/cell
        prog1 = ProgressDialog("Script Processing..", 0, len(ordered))
        ncell = len(ordered)
        for nc, cell in enumerate(ordered):
            if prog1.wasCanceled():
                break
            presetDict = {}
            thiscell = self.script['Cells'][cell]
            #print 'processing cell: %s' % thiscell
            if thiscell['include'] is False:  # skip this cell
                try:
                    print('Skipped: %s, reason:%s' %
                          (cell, thiscell['reason']))
                except:
                    raise ValueError(
                        'cell %s has no tag "reason" but "include" is False' %
                        cell)

                continue
            sortedkeys = sorted(thiscell['choice'].keys()
                                )  # sort by order of recording (# on protocol)
            prog1.setValue(nc / ncell)
            #            prog2 = ProgressDialog("Cell Processing..%s" , 0, len(sortedkeys)):
            for p in sortedkeys:
                if thiscell['choice'][p] not in self.script[
                        'datafilter']:  # pick out steady-state conditions
                    print('p: %s not in data: ' % (thiscell['choice'][p]),
                          self.script['datafilter'])
                    continue
                # print 'working on %s' % thiscell['choice'][p]
                pr = self.script[
                    'protocol'] + '_' + p  # add the underscore here
                if cell[-1] in trailingchars:  # check last letter - if not a number clip it
                    cell_file = cell[:-1]
                else:
                    cell_file = cell
                fn = os.path.join(cell_file, pr)
                #dm_selected_file = self.dataManager().selectedFile().name()
                dm_selected_file = self.script['directory']
                fullpath = os.path.join(dm_selected_file, fn)
                file_ok = os.path.exists(fullpath)
                if not file_ok:  # get the directory handle and take it from there
                    print('File is not ok: %s' % fullpath)
                    continue

                m = thiscell['choice'][p]  # get the tag for the manipulation
                presetDict['Choices'] = thiscell['choice'][p]
                if 'genotype' in thiscell.keys():
                    presetDict['Genotype'] = thiscell['genotype']
                else:
                    presetDict['Genotype'] = 'Unknown'
                if 'Celltype' in thiscell.keys():
                    presetDict['Celltype'] = thiscell['Celltype']
                else:
                    presetDict['Celltype'] = 'Unknown'
                if 'spikethresh' in thiscell.keys():
                    presetDict['SpikeThreshold'] = thiscell['spikethresh']
                if 'bridgeCorrection' in thiscell.keys():
                    presetDict['bridgeCorrection'] = thiscell[
                        'bridgeCorrection']
                else:
                    presetDict['bridgeCorrection'] = None

                dh = self.dataManager().manager.dirHandle(fullpath)
                if not self.loadFile(
                    [dh], analyze=False, bridge=presetDict['bridgeCorrection']
                ):  # note: must pass a list of dh; don't let analyisis run at end
                    print('Failed to load requested file: ', fullpath)
                    continue  # skip bad sets of records...
                if 'datamode' in thiscell.keys():
                    self.clamps.data_mode = thiscell['datamode']
                self.auto_updater = False
                self.get_script_analysisPars(self.script, thiscell)
                self.analysis(
                    presets=presetDict)  # call the caller's analysis routine

                if 'addtoDB' in self.script.keys():
                    if self.script[
                            'addtoDB'] is True and self.dbUpdate is not None:
                        self.dbUpdate()  # call routine in parent

                ptxt = self.printAnalysis(printnow=False,
                                          script_header=script_header,
                                          copytoclipboard=False)
                self.textout += ptxt + '\n'
                script_header = False

                DataManager.cleanup()
                del dh
                gc.collect()

        print(self.textout)
        self.auto_updater = True  # restore function