def _mergeH5(self, parts, outfilename): shutil.copy(parts[0], outfilename) with h5py.File(outfilename) as fout: for entry in NexusTools.getNXClassGroups(fout, '/', [u'NXentry']): for process in NexusTools.getNXClassGroups(fout, entry.name, [u'NXprocess']): for results in NexusTools.getNXClassGroups(fout, process.name, [u'NXcollection']): for dataout in NexusTools.getNXClassGroups(fout, results.name, [u'NXdata']): for part in parts[1:]: with h5py.File(part) as fin: try: datain = fin[dataout.name] except KeyError: _logger.error('%s does not have %s', part, repr(dataout.name)) continue for datasetname in dataout: self._fillPartial(dataout[datasetname], datain[datasetname], maxdims=2)
def _mergeH5(self, parts, outfilename): shutil.copy(parts[0], outfilename) with h5py.File(outfilename, mode='a') as fout: for entry in NexusTools.getNXClassGroups(fout, '/', [u'NXentry']): for process in NexusTools.getNXClassGroups( fout, entry.name, [u'NXprocess']): for results in NexusTools.getNXClassGroups( fout, process.name, [u'NXcollection']): for dataout in NexusTools.getNXClassGroups( fout, results.name, [u'NXdata']): for part in parts[1:]: with h5py.File(part, mode='r') as fin: try: datain = fin[dataout.name] except KeyError: _logger.error('%s does not have %s', part, repr(dataout.name)) continue for datasetname in dataout: self._fillPartial(dataout[datasetname], datain[datasetname], maxdims=2)
def hdf5Slot(self, ddict): _logger.debug("hdf5Slot %s" % ddict) entryName = NexusTools.getEntryName(ddict['name']) currentEntry = "%s::%s" % (ddict['file'], entryName) if (currentEntry != self._lastEntry) and not self._BUTTONS: self._lastEntry = None cntList = [] mcaList = [] aliasList = [] measurement = None scanned = [] mcaList = [] if posixpath.dirname(entryName) != entryName: h5file = HDF5Widget.h5open(ddict['file']) try: measurement = NexusTools.getMeasurementGroup(h5file, ddict['name']) scanned = NexusTools.getScannedPositioners(h5file, ddict['name']) if measurement is not None: measurement = [item.name for key,item in measurement.items() \ if self._isNumeric(item)] try: # case insensitive sorting of measurement if sys.version_info > (3, 3): measurement.sort(key=str.casefold) except: _logger.error("Cannot apply sorting %s" % sys.exc_info()[1]) if self._mca: mcaList = NexusTools.getMcaList(h5file, entryName) finally: h5file.close() h5file = None del h5file for i in range(len(scanned)): key = scanned[i] cntList.append(key) aliasList.append(posixpath.basename(key)) if measurement is not None: for key in measurement: if key not in cntList: cntList.append(key) aliasList.append(posixpath.basename(key)) cleanedCntList = [] for key in cntList: root = key.split('/') root = "/" + root[1] if len(key) == len(root): cleanedCntList.append(key) else: cleanedCntList.append(key[len(root):]) self._autoAliasList = aliasList self._autoCntList = cleanedCntList _logger.info("building autoTable") self.autoTable.build(self._autoCntList, self._autoAliasList, selection=self._lastCntSelection) currentTab = qt.safe_str(self.tableTab.tabText( \ self.tableTab.currentIndex())) if self._mca: mcaAliasList = [] cleanedMcaList = [] for key in mcaList: root = key.split('/') root = "/" + root[1] if len(key) == len(root): cleanedMcaList.append(key) else: cleanedMcaList.append(key[len(root):]) mcaAliasList.append(posixpath.basename(key)) self.mcaTable.build(cleanedMcaList, mcaAliasList) nTabs = self.tableTab.count() if (len(mcaList) > 0) and (nTabs < 3): self.tableTab.insertTab(2, self.mcaTable, "MCA") elif (len(mcaList)==0) and (nTabs > 2): self.tableTab.removeTab(2) _logger.debug("currentTab = %s", currentTab) if currentTab != "USER": if (len(mcaList) > 0) and (len(cntList) == 0): idx = self.tableTabOrder.index("MCA") self.tableTab.setCurrentIndex(idx) _logger.debug("setting tab = %s MCA", idx) elif (len(mcaList) == 0) and (len(cntList) > 0): idx = self.tableTabOrder.index("AUTO") self.tableTab.setCurrentIndex(idx) _logger.debug("setting tab = %s AUTO", idx) self._lastEntry = currentEntry if ddict['event'] == 'itemClicked': if ddict['mouse'] == "right": return self.itemRightClickedSlot(ddict) if ddict['mouse'] == "left": # If parent is root do it even if not NXentry?? if ddict['type'] in ['NXentry', 'Entry']: if not self._BUTTONS: auto = self.actions.getConfiguration()["auto"] if auto == "ADD": self._addAction() elif auto == "REPLACE": self._replaceAction() if ddict['event'] == "itemDoubleClicked": if ddict['type'] in ['Dataset']: currentIndex = self.tableTab.currentIndex() text = safe_str(self.tableTab.tabText(currentIndex)) if text.upper() != "USER": if currentIndex == 0: self.tableTab.setCurrentIndex(1) else: self.tableTab.setCurrentIndex(0) if not self._isNumericType(ddict['dtype']): _logger.debug("string like %s", ddict['dtype']) else: root = ddict['name'].split('/') root = "/" + root[1] if len(ddict['name']) == len(root): cnt = ddict['name'] else: cnt = ddict['name'][len(root):] if cnt not in self._cntList: self._cntList.append(cnt) basename = posixpath.basename(cnt) if basename not in self._aliasList: self._aliasList.append(basename) else: self._aliasList.append(cnt) self.cntTable.build(self._cntList, self._aliasList) elif (ddict['color'] == qt.Qt.blue) and ("silx" in sys.modules): # there is an action to be applied self.showInfoWidget(ddict["file"], ddict["name"], dset=False) elif ddict['type'] in ['NXentry', 'Entry']: if self._lastAction is None: return action, selectionType = self._lastAction.split() if action == 'REMOVE': action = 'ADD' ddict['action'] = "%s %s" % (action, selectionType) self.buttonsSlot(ddict) else: if self.data is not None: name = ddict['name'] filename = ddict['file'] fileIndex = self.data.sourceName.index(filename) phynxFile = self.data._sourceObjectList[fileIndex] dataset = phynxFile[name] if isinstance(dataset, h5py.Dataset): root = ddict['name'].split('/') root = "/" + root[1] cnt = ddict['name'].split(root)[-1] if cnt not in self._cntList: _logger.debug("USING SECOND WAY") self._cntList.append(cnt) basename = posixpath.basename(cnt) if basename not in self._aliasList: self._aliasList.append(basename) else: self._aliasList.append(cnt) self.cntTable.build(self._cntList, self._aliasList) return _logger.debug("Unhandled item type: %s", ddict['dtype'])
def loadFileList(self, filelist, selection, scanlist=None): """ loadFileList(self, filelist, y, scanlist=None, monitor=None, x=None) filelist is the list of file names belonging to the stack selection is a dictionary with the keys x, y, m. x is the path to the x data (the channels) in the spectrum, without the first level "directory". It is unused (for now). y is the path to the 1D data (the counts) in the spectrum, without the first level "directory" m is the path to the normalizing data (I0 or whatever) without the first level "directory". scanlist is the list of first level "directories" containing the 1D data Example: The actual path has the form: /whatever1/whatever2/counts That means scanlist = ["/whatever1"] and selection['y'] = "/whatever2/counts" """ _logger.info("filelist = %s", filelist) _logger.info("selection = %s", selection) _logger.info("scanlist = %s", scanlist) # all the files in the same source hdfStack = NexusDataSource.NexusDataSource(filelist) # if there is more than one file, it is assumed all the files have # the same structure. tmpHdf = hdfStack._sourceObjectList[0] entryNames = [] for key in tmpHdf["/"].keys(): try: if isinstance(tmpHdf["/" + key], h5py.Group): entryNames.append(key) except KeyError: _logger.info("Broken link with key? <%s>" % key) # built the selection in terms of HDF terms # for the time being xSelectionList = selection.get('x', None) if xSelectionList == []: xSelectionList = None if xSelectionList is not None: if type(xSelectionList) != type([]): xSelectionList = [xSelectionList] if len(xSelectionList): xSelection = xSelectionList[0] else: xSelection = None else: xSelection = None # only one y is taken ySelection = selection['y'] if type(ySelection) == type([]): ySelectionList = list(ySelection) ySelection = ySelection[0] else: ySelectionList = [ySelection] # monitor selection mSelection = selection.get('m', None) if mSelection not in [None, []]: if type(mSelection) != type([]): mSelection = [mSelection] if type(mSelection) == type([]): if len(mSelection): mSelection = mSelection[0] else: mSelection = None else: mSelection = None USE_JUST_KEYS = False # deal with the pathological case where the scanlist corresponds # to a selected top level dataset if len(entryNames) == 0: if scanlist is not None: if (ySelection in scanlist) or \ (xSelection in scanlist) or \ (mSelection in scanlist): scanlist = None USE_JUST_KEYS = True else: USE_JUST_KEYS = True elif len(entryNames) == 1: # deal with the SOLEIL case of one entry but with different name # in different files USE_JUST_KEYS = True elif scanlist in [None, []]: USE_JUST_KEYS = True if USE_JUST_KEYS: # if the scanlist is None, it is assumed we are interested on all # the scans containing the selection, not that all the scans # contain the selection. scanlist = [] if 0: JUST_KEYS = False #expect same entry names in the files #Unfortunately this does not work for SOLEIL for entry in entryNames: path = "/" + entry + ySelection dirname = posixpath.dirname(path) base = posixpath.basename(path) try: file_entry = tmpHdf[dirname] if base in file_entry.keys(): scanlist.append(entry) except: pass else: JUST_KEYS = True #expect same structure in the files even if the #names are different (SOLEIL ...) if len(entryNames): i = 0 for entry in entryNames: i += 1 path = "/" + entry + ySelection dirname = posixpath.dirname(path) base = posixpath.basename(path) try: file_entry = tmpHdf[dirname] if hasattr(file_entry, "keys"): if base in file_entry.keys(): # this is the case of a selection inside a group scanlist.append("1.%d" % i) except KeyError: _logger.warning("%s not in file, ignoring.", dirname) if not len(scanlist): if not ySelection.startswith("/"): path = "/" + ySelection else: path = ySelection dirname = posixpath.dirname(path) base = posixpath.basename(path) try: if dirname in tmpHdf["/"]: # this is the case of a dataset at top plevel # or having given the complete path if base in tmpHdf[dirname]: JUST_KEYS = False scanlist.append("") elif base in file_entry.keys(): JUST_KEYS = False scanlist.append("") except: #it will crash later on pass else: JUST_KEYS = False scanlist.append("") else: try: number, order = [int(x) for x in scanlist[0].split(".")] JUST_KEYS = True except: JUST_KEYS = False if not JUST_KEYS: for scan in scanlist: if scan.startswith("/"): t = scan[1:] else: t = scan if t not in entryNames: raise ValueError("Entry %s not in file" % scan) nFiles = len(filelist) nScans = len(scanlist) if JUST_KEYS: if not nScans: raise IOError("No entry contains the required data") _logger.debug("Retained number of files = %d", nFiles) _logger.debug("Retained number of scans = %d", nScans) # Now is to decide the number of mca ... # I assume all the scans contain the same number of mca if JUST_KEYS: path = "/" + entryNames[int(scanlist[0].split(".")[-1]) - 1] + ySelection if mSelection is not None: mpath = "/" + entryNames[int(scanlist[0].split(".")[-1]) - 1] + mSelection if xSelectionList is not None: xpathList = [] for xSelection in xSelectionList: xpath = "/" + entryNames[int(scanlist[0].split(".")[-1]) - 1] + xSelection xpathList.append(xpath) else: path = scanlist[0] + ySelection if mSelection is not None: mpath = scanlist[0] + mSelection if xSelectionList is not None: xpathList = [] for xSelection in xSelectionList: xpath = scanlist[0] + xSelection xpathList.append(xpath) yDataset = tmpHdf[path] if (self.__dtype is None) or (mSelection is not None): self.__dtype = yDataset.dtype if self.__dtype in [numpy.int16, numpy.uint16]: self.__dtype = numpy.float32 elif self.__dtype in [numpy.int32, numpy.uint32]: if mSelection: self.__dtype = numpy.float32 else: self.__dtype = numpy.float64 elif self.__dtype not in [ numpy.float16, numpy.float32, numpy.float64 ]: # Some datasets form CLS (origin APS?) arrive as data format # equal to ">u2" and are not triggered as integer types _logger.debug("Not basic dataset type %s", self.__dtype) if ("%s" % self.__dtype).endswith("2"): self.__dtype = numpy.float32 else: if mSelection: self.__dtype = numpy.float32 else: self.__dtype = numpy.float64 # figure out the shape of the stack shape = yDataset.shape mcaIndex = selection.get('index', len(shape) - 1) if mcaIndex == -1: mcaIndex = len(shape) - 1 _logger.debug("mcaIndex = %d", mcaIndex) considerAsImages = False dim0, dim1, mcaDim = self.getDimensions(nFiles, nScans, shape, index=mcaIndex) try: if self.__dtype in [numpy.float32, numpy.int32]: bytefactor = 4 elif self.__dtype in [numpy.int16, numpy.uint16]: bytefactor = 2 elif self.__dtype in [numpy.int8, numpy.uint8]: bytefactor = 1 else: bytefactor = 8 neededMegaBytes = nFiles * dim0 * dim1 * (mcaDim * bytefactor / (1024 * 1024.)) _logger.info("Using %d bytes per item" % bytefactor) _logger.info("Needed %d Megabytes" % neededMegaBytes) physicalMemory = None if hasattr(PhysicalMemory, "getAvailablePhysicalMemoryOrNone"): physicalMemory = PhysicalMemory.getAvailablePhysicalMemoryOrNone( ) if not physicalMemory: physicalMemory = PhysicalMemory.getPhysicalMemoryOrNone() else: _logger.info("Available physical memory %.1f GBytes" % \ (physicalMemory/(1024*1024*1024.))) if physicalMemory is None: # 6 Gigabytes of available memory # should be a good compromise in 2018 physicalMemory = 6000 _logger.info("Assumed physical memory %.1f MBytes" % physicalMemory) else: physicalMemory /= (1024 * 1024.) _logger.info("Using physical memory %.1f GBytes" % (physicalMemory / 1024)) if (neededMegaBytes > (0.95*physicalMemory))\ and (nFiles == 1) and (len(shape) == 3): if self.__dtype0 is None: if (bytefactor == 8) and (neededMegaBytes < (2 * physicalMemory)): # try reading as float32 print("Forcing the use of float32 data") self.__dtype = numpy.float32 else: raise MemoryError("Force dynamic loading") else: raise MemoryError("Force dynamic loading") if (mcaIndex == 0) and (nFiles == 1) and (nScans == 1): #keep the original arrangement but in memory self.data = numpy.zeros(yDataset.shape, self.__dtype) considerAsImages = True else: # force arrangement as spectra self.data = numpy.zeros((dim0, dim1, mcaDim), self.__dtype) DONE = False except (MemoryError, ValueError): # some versions report ValueError instead of MemoryError if (nFiles == 1) and (len(shape) == 3): _logger.warning("Attempting dynamic loading") if mSelection is not None: _logger.warning("Ignoring monitor") self.data = yDataset if mSelection is not None: mdtype = tmpHdf[mpath].dtype if mdtype not in [numpy.float64, numpy.float32]: mdtype = numpy.float64 mDataset = numpy.asarray(tmpHdf[mpath], dtype=mdtype) self.monitor = [mDataset] if xSelectionList is not None: if len(xpathList) == 1: xpath = xpathList[0] xDataset = tmpHdf[xpath][()] self.x = [xDataset] if h5py.version.version < '2.0': #prevent automatic closing keeping a reference #to the open file self._fileReference = hdfStack DONE = True else: # what to do if the number of dimensions is only 2? raise # get the positioners information associated to the path positioners = {} try: positionersGroup = NexusTools.getPositionersGroup(tmpHdf, path) for motorName, motorValues in positionersGroup.items(): positioners[motorName] = motorValues[()] except: positionersGroup = None positioners = {} # get the mca information associated to the path mcaObjectPaths = NexusTools.getMcaObjectPaths(tmpHdf, path) _time = None _calibration = None _channels = None if considerAsImages: self._pathHasRelevantInfo = False else: numberOfRelevantInfoKeys = 0 for objectPath in mcaObjectPaths: if objectPath not in ["counts", "target"]: numberOfRelevantInfoKeys += 1 if numberOfRelevantInfoKeys: # not just "counts" or "target" self._pathHasRelevantInfo = True if "live_time" in mcaObjectPaths: if DONE: # hopefully it will fit into memory if mcaObjectPaths["live_time"] in tmpHdf: _time = tmpHdf[mcaObjectPaths["live_time"]][()] elif "::" in mcaObjectPaths["live_time"]: tmpFileName, tmpDatasetPath = \ mcaObjectPaths["live_time"].split("::") with h5py.File(tmpFileName, "r") as tmpH5: _time = tmpH5[tmpDatasetPath][()] else: del mcaObjectPaths["live_time"] else: # we have to have as many live times as MCA spectra _time = numpy.zeros( \ (self.data.shape[0] * self.data.shape[1]), dtype=numpy.float64) elif "elapsed_time" in mcaObjectPaths: if DONE: # hopefully it will fit into memory if mcaObjectPaths["elapsed_time"] in tmpHdf: _time = \ tmpHdf[mcaObjectPaths["elapsed_time"]][()] elif "::" in mcaObjectPaths["elapsed_time"]: tmpFileName, tmpDatasetPath = \ mcaObjectPaths["elapsed_time"].split("::") with h5py.File(tmpFileName, "r") as tmpH5: _time = tmpH5[tmpDatasetPath][()] else: del mcaObjectPaths["elapsed_time"] else: # we have to have as many elpased times as MCA spectra _time = numpy.zeros( (self.data.shape[0] * self.data.shape[1]), numpy.float32) if "calibration" in mcaObjectPaths: if mcaObjectPaths["calibration"] in tmpHdf: _calibration = \ tmpHdf[mcaObjectPaths["calibration"]][()] elif "::" in mcaObjectPaths["calibration"]: tmpFileName, tmpDatasetPath = \ mcaObjectPaths["calibration"].split("::") with h5py.File(tmpFileName, "r") as tmpH5: _calibration = tmpH5[tmpDatasetPath][()] else: del mcaObjectPaths["calibration"] if "channels" in mcaObjectPaths: if mcaObjectPaths["channels"] in tmpHdf: _channels = \ tmpHdf[mcaObjectPaths["channels"]][()] elif "::" in mcaObjectPaths["channels"]: tmpFileName, tmpDatasetPath = \ mcaObjectPaths["channels"].split("::") with h5py.File(tmpFileName, "r") as tmpH5: _channels = tmpH5[tmpDatasetPath][()] else: del mcaObjectPaths["channels"] else: self._pathHasRelevantInfo = False if (not DONE) and (not considerAsImages): _logger.info("Data in memory as spectra") self.info["McaIndex"] = 2 n = 0 if dim0 == 1: self.onBegin(dim1) else: self.onBegin(dim0) self.incrProgressBar = 0 for hdf in hdfStack._sourceObjectList: entryNames = list(hdf["/"].keys()) goodEntryNames = [] for entry in entryNames: tmpPath = "/" + entry try: if hasattr(hdf[tmpPath], "keys"): goodEntryNames.append(entry) except KeyError: _logger.info("Broken link with key? <%s>" % tmpPath) for scan in scanlist: IN_MEMORY = None nStart = n for ySelection in ySelectionList: n = nStart if JUST_KEYS: entryName = goodEntryNames[ int(scan.split(".")[-1]) - 1] path = entryName + ySelection if mSelection is not None: mpath = entryName + mSelection mdtype = hdf[mpath].dtype if mdtype not in [ numpy.float64, numpy.float32 ]: mdtype = numpy.float64 mDataset = numpy.asarray(hdf[mpath], dtype=mdtype) if xSelectionList is not None: xDatasetList = [] for xSelection in xSelectionList: xpath = entryName + xSelection xDataset = hdf[xpath][()] xDatasetList.append(xDataset) else: path = scan + ySelection if mSelection is not None: mpath = scan + mSelection mdtype = hdf[mpath].dtype if mdtype not in [ numpy.float64, numpy.float32 ]: mdtype = numpy.float64 mDataset = numpy.asarray(hdf[mpath], dtype=mdtype) if xSelectionList is not None: xDatasetList = [] for xSelection in xSelectionList: xpath = scan + xSelection xDataset = hdf[xpath][()] xDatasetList.append(xDataset) try: yDataset = hdf[path] tmpShape = yDataset.shape totalBytes = numpy.ones((1, ), yDataset.dtype).itemsize for nItems in tmpShape: totalBytes *= nItems # should one be conservative or just try? if (totalBytes / (1024. * 1024.)) > (0.4 * physicalMemory): _logger.info( "Force dynamic loading of spectra") #read from disk IN_MEMORY = False else: #read the data into memory _logger.info( "Attempt to load whole map into memory") yDataset = hdf[path][()] IN_MEMORY = True except (MemoryError, ValueError): _logger.info("Dynamic loading of spectra") yDataset = hdf[path] IN_MEMORY = False nMcaInYDataset = 1 for dim in yDataset.shape: nMcaInYDataset *= dim nMcaInYDataset = int(nMcaInYDataset / mcaDim) timeData = None if _time is not None: if "live_time" in mcaObjectPaths: # it is assumed that all have the same structure!!! timePath = NexusTools.getMcaObjectPaths( hdf, path)["live_time"] elif "elapsed_time" in mcaObjectPaths: timePath = NexusTools.getMcaObjectPaths( hdf, path)["elapsed_time"] if timePath in hdf: timeData = hdf[timePath][()] elif "::" in timePath: externalFile, externalPath = timePath.split( "::") with h5py.File(externalFile, "r") as timeHdf: timeData = timeHdf[externalPath][()] if mcaIndex != 0: if IN_MEMORY: yDataset.shape = -1, mcaDim if mSelection is not None: case = -1 nMonitorData = 1 for v in mDataset.shape: nMonitorData *= v if nMonitorData == nMcaInYDataset: mDataset.shape = nMcaInYDataset case = 0 elif nMonitorData == (nMcaInYDataset * mcaDim): case = 1 mDataset.shape = nMcaInYDataset, mcaDim if case == -1: raise ValueError(\ "I do not know how to handle this monitor data") if timeData is not None: case = -1 nTimeData = 1 for v in timeData.shape: nTimeData *= v if nTimeData == nMcaInYDataset: timeData.shape = nMcaInYDataset case = 0 _time[nStart:nStart + nMcaInYDataset] += timeData if case == -1: _logger.warning( "I do not know how to handle this time data" ) _logger.warning( "Ignoring time information") _time = None if (len(yDataset.shape) == 3) and\ (dim1 == yDataset.shape[1]): mca = 0 deltaI = int(yDataset.shape[1] / dim1) for ii in range(yDataset.shape[0]): i = int(n / dim1) yData = yDataset[ii:(ii + 1)] yData.shape = -1, mcaDim if mSelection is not None: if case == 0: mData = numpy.outer( mDataset[mca:(mca + dim1)], numpy.ones((mcaDim))) self.data[i, :, :] += yData / mData elif case == 1: mData = mDataset[mca:(mca + dim1), :] mData.shape = -1, mcaDim self.data[i, :, :] += yData / mData else: self.data[i:(i + deltaI), :] += yData n += yDataset.shape[1] mca += dim1 else: for mca in range(nMcaInYDataset): i = int(n / dim1) j = n % dim1 if len(yDataset.shape) == 3: ii = int(mca / yDataset.shape[1]) jj = mca % yDataset.shape[1] yData = yDataset[ii, jj] elif len(yDataset.shape) == 2: yData = yDataset[mca, :] elif len(yDataset.shape) == 1: yData = yDataset if mSelection is not None: if case == 0: self.data[ i, j, :] += yData / mDataset[mca] elif case == 1: self.data[ i, j, :] += yData / mDataset[ mca, :] else: self.data[i, j, :] += yData n += 1
def loadFileList(self, filelist, selection, scanlist=None): """ loadFileList(self, filelist, y, scanlist=None, monitor=None, x=None) filelist is the list of file names belonging to the stack selection is a dictionary with the keys x, y, m. x is the path to the x data (the channels) in the spectrum, without the first level "directory". It is unused (for now). y is the path to the 1D data (the counts) in the spectrum, without the first level "directory" m is the path to the normalizing data (I0 or whatever) without the first level "directory". scanlist is the list of first level "directories" containing the 1D data Example: The actual path has the form: /whatever1/whatever2/counts That means scanlist = ["/whatever1"] and selection['y'] = "/whatever2/counts" """ _logger.debug("filelist = %s", filelist) _logger.debug("selection = %s", selection) _logger.debug("scanlist = %s", scanlist) # all the files in the same source hdfStack = NexusDataSource.NexusDataSource(filelist) # if there is more than one file, it is assumed all the files have # the same structure. tmpHdf = hdfStack._sourceObjectList[0] entryNames = [] for key in tmpHdf["/"].keys(): if isinstance(tmpHdf["/"+key], h5py.Group): entryNames.append(key) # built the selection in terms of HDF terms # for the time being, only the first item in x selection used xSelection = selection.get('x', None) if xSelection is not None: if type(xSelection) != type([]): xSelection = [xSelection] if type(xSelection) == type([]): if len(xSelection): xSelection = xSelection[0] else: xSelection = None else: xSelection = None # only one y is taken ySelection = selection['y'] if type(ySelection) == type([]): ySelectionList = list(ySelection) ySelection = ySelection[0] else: ySelectionList = [ySelection] # monitor selection mSelection = selection.get('m', None) if mSelection not in [None, []]: if type(mSelection) != type([]): mSelection = [mSelection] if type(mSelection) == type([]): if len(mSelection): mSelection = mSelection[0] else: mSelection = None else: mSelection = None USE_JUST_KEYS = False # deal with the pathological case where the scanlist corresponds # to a selected top level dataset if len(entryNames) == 0: if scanlist is not None: if (ySelection in scanlist) or \ (xSelection in scanlist) or \ (mSelection in scanlist): scanlist = None USE_JUST_KEYS = True else: USE_JUST_KEYS = True elif len(entryNames) == 1: # deal with the SOLEIL case of one entry but with different name # in different files USE_JUST_KEYS = True elif scanlist in [None, []]: USE_JUST_KEYS = True if USE_JUST_KEYS: # if the scanlist is None, it is assumed we are interested on all # the scans containing the selection, not that all the scans # contain the selection. scanlist = [] if 0: JUST_KEYS = False #expect same entry names in the files #Unfortunately this does not work for SOLEIL for entry in entryNames: path = "/" + entry + ySelection dirname = posixpath.dirname(path) base = posixpath.basename(path) try: file_entry = tmpHdf[dirname] if base in file_entry.keys(): scanlist.append(entry) except: pass else: JUST_KEYS = True #expect same structure in the files even if the #names are different (SOLEIL ...) if len(entryNames): i = 0 for entry in entryNames: i += 1 path = "/" + entry + ySelection dirname = posixpath.dirname(path) base = posixpath.basename(path) try: file_entry = tmpHdf[dirname] if hasattr(file_entry, "keys"): if base in file_entry.keys(): # this is the case of a selection inside a group scanlist.append("1.%d" % i) except KeyError: _logger.warning("%s not in file, ignoring.", dirname) if not len(scanlist): if not ySelection.startswith("/"): path = "/" + ySelection else: path = ySelection dirname = posixpath.dirname(path) base = posixpath.basename(path) try: if dirname in tmpHdf["/"]: # this is the case of a dataset at top plevel # or having given the complete path if base in tmpHdf[dirname]: JUST_KEYS = False scanlist.append("") elif base in file_entry.keys(): JUST_KEYS = False scanlist.append("") except: #it will crash later on pass else: JUST_KEYS = False scanlist.append("") else: try: number, order = [int(x) for x in scanlist[0].split(".")] JUST_KEYS = True except: JUST_KEYS = False if not JUST_KEYS: for scan in scanlist: if scan.startswith("/"): t = scan[1:] else: t = scan if t not in entryNames: raise ValueError("Entry %s not in file" % scan) nFiles = len(filelist) nScans = len(scanlist) if JUST_KEYS: if not nScans: raise IOError("No entry contains the required data") _logger.debug("Retained number of files = %d", nFiles) _logger.debug("Retained number of scans = %d", nScans) # Now is to decide the number of mca ... # I assume all the scans contain the same number of mca if JUST_KEYS: path = "/" + entryNames[int(scanlist[0].split(".")[-1])-1] + ySelection if mSelection is not None: mpath = "/" + entryNames[int(scanlist[0].split(".")[-1])-1] + mSelection if xSelection is not None: xpath = "/" + entryNames[int(scanlist[0].split(".")[-1])-1] + xSelection else: path = scanlist[0] + ySelection if mSelection is not None: mpath = scanlist[0] + mSelection if xSelection is not None: xpath = scanlist[0] + xSelection yDataset = tmpHdf[path] if (self.__dtype is None) or (mSelection is not None): self.__dtype = yDataset.dtype if self.__dtype in [numpy.int16, numpy.uint16]: self.__dtype = numpy.float32 elif self.__dtype in [numpy.int32, numpy.uint32]: if mSelection: self.__dtype = numpy.float32 else: self.__dtype = numpy.float64 elif self.__dtype not in [numpy.float16, numpy.float32, numpy.float64]: # Some datasets form CLS (origin APS?) arrive as data format # equal to ">u2" and are not triggered as integer types _logger.debug("Not basic dataset type %s", self.__dtype) if ("%s" % self.__dtype).endswith("2"): self.__dtype = numpy.float32 else: if mSelection: self.__dtype = numpy.float32 else: self.__dtype = numpy.float64 # figure out the shape of the stack shape = yDataset.shape mcaIndex = selection.get('index', len(shape)-1) if mcaIndex == -1: mcaIndex = len(shape) - 1 _logger.debug("mcaIndex = %d", mcaIndex) considerAsImages = False dim0, dim1, mcaDim = self.getDimensions(nFiles, nScans, shape, index=mcaIndex) try: if self.__dtype in [numpy.float32, numpy.int32]: bytefactor = 4 elif self.__dtype in [numpy.int16, numpy.uint16]: bytefactor = 2 elif self.__dtype in [numpy.int8, numpy.uint8]: bytefactor = 1 else: bytefactor = 8 neededMegaBytes = nFiles * dim0 * dim1 * (mcaDim * bytefactor/(1024*1024.)) _logger.info("Using %d bytes per item" % bytefactor) _logger.info("Needed %d Megabytes" % neededMegaBytes) physicalMemory = None if hasattr(PhysicalMemory, "getAvailablePhysicalMemoryOrNone"): physicalMemory = PhysicalMemory.getAvailablePhysicalMemoryOrNone() if not physicalMemory: physicalMemory = PhysicalMemory.getPhysicalMemoryOrNone() else: _logger.info("Available physical memory %.1f GBytes" % \ (physicalMemory/(1024*1024*1024.))) if physicalMemory is None: # 6 Gigabytes of available memory # should be a good compromise in 2018 physicalMemory = 6000 _logger.info("Assumed physical memory %.1f MBytes" % physicalMemory) else: physicalMemory /= (1024*1024.) _logger.info("Using physical memory %.1f GBytes" % (physicalMemory/1024)) if (neededMegaBytes > (0.95*physicalMemory))\ and (nFiles == 1) and (len(shape) == 3): if self.__dtype0 is None: if (bytefactor == 8) and (neededMegaBytes < (2*physicalMemory)): # try reading as float32 print("Forcing the use of float32 data") self.__dtype = numpy.float32 else: raise MemoryError("Force dynamic loading") else: raise MemoryError("Force dynamic loading") if (mcaIndex == 0) and ( nFiles == 1) and (nScans == 1): #keep the original arrangement but in memory self.data = numpy.zeros(yDataset.shape, self.__dtype) considerAsImages = True else: # force arrangement as spectra self.data = numpy.zeros((dim0, dim1, mcaDim), self.__dtype) DONE = False except (MemoryError, ValueError): # some versions report ValueError instead of MemoryError if (nFiles == 1) and (len(shape) == 3): _logger.warning("Attempting dynamic loading") if mSelection is not None: _logger.warning("Ignoring monitor") self.data = yDataset if mSelection is not None: mdtype = tmpHdf[mpath].dtype if mdtype not in [numpy.float64, numpy.float32]: mdtype = numpy.float64 mDataset = numpy.asarray(tmpHdf[mpath], dtype=mdtype) self.monitor = [mDataset] if xSelection is not None: xDataset = tmpHdf[xpath][()] self.x = [xDataset] if h5py.version.version < '2.0': #prevent automatic closing keeping a reference #to the open file self._fileReference = hdfStack DONE = True else: # what to do if the number of dimensions is only 2? raise # get the mca information associated to the path mcaObjectPaths = NexusTools.getMcaObjectPaths(tmpHdf, path) _time = None _calibration = None _channels = None if considerAsImages: self._pathHasRelevantInfo = False else: if len(list(mcaObjectPaths.keys())) > 1: # not just "counts" self._pathHasRelevantInfo = True if "live_time" in mcaObjectPaths: if DONE: # hopefully it will fit into memory if mcaObjectPaths["live_time"] in tmpHdf: _time = tmpHdf[mcaObjectPaths["live_time"]][()] elif "::" in mcaObjectPaths["live_time"]: tmpFileName, tmpDatasetPath = \ mcaObjectPaths["live_time"].split("::") with h5py.File(tmpFileName, "r") as tmpH5: _time = tmpH5[tmpDatasetPath][()] else: del mcaObjectPaths["live_time"] else: # we have to have as many live times as MCA spectra _time = numpy.zeros( \ (self.data.shape[0] * self.data.shape[1]), dtype=numpy.float64) elif "elapsed_time" in mcaObjectPaths: if DONE: # hopefully it will fit into memory if mcaObjectPaths["elapsed_time"] in tmpHdf: _time = \ tmpHdf[mcaObjectPaths["elapsed_time"]][()] elif "::" in mcaObjectPaths["elapsed_time"]: tmpFileName, tmpDatasetPath = \ mcaObjectPaths["elapsed_time"].split("::") with h5py.File(tmpFileName, "r") as tmpH5: _time = tmpH5[tmpDatasetPath][()] else: del mcaObjectPaths["elapsed_time"] else: # we have to have as many elpased times as MCA spectra _time = numpy.zeros((self.data.shape[0] * self.data.shape[1]), numpy.float32) if "calibration" in mcaObjectPaths: if mcaObjectPaths["calibration"] in tmpHdf: _calibration = \ tmpHdf[mcaObjectPaths["calibration"]][()] elif "::" in mcaObjectPaths["calibration"]: tmpFileName, tmpDatasetPath = \ mcaObjectPaths["calibration"].split("::") with h5py.File(tmpFileName, "r") as tmpH5: _calibration = tmpH5[tmpDatasetPath][()] else: del mcaObjectPaths["calibration"] if "channels" in mcaObjectPaths: if mcaObjectPaths["channels"] in tmpHdf: _channels = \ tmpHdf[mcaObjectPaths["channels"]][()] elif "::" in mcaObjectPaths["channels"]: tmpFileName, tmpDatasetPath = \ mcaObjectPaths["channels"].split("::") with h5py.File(tmpFileName, "r") as tmpH5: _channels = tmpH5[tmpDatasetPath][()] else: del mcaObjectPaths["channels"] else: self._pathHasRelevantInfo = False if (not DONE) and (not considerAsImages): _logger.info("Data in memory as spectra") self.info["McaIndex"] = 2 n = 0 if dim0 == 1: self.onBegin(dim1) else: self.onBegin(dim0) self.incrProgressBar=0 for hdf in hdfStack._sourceObjectList: entryNames = list(hdf["/"].keys()) goodEntryNames = [] for entry in entryNames: tmpPath = "/" + entry if hasattr(hdf[tmpPath], "keys"): goodEntryNames.append(entry) for scan in scanlist: IN_MEMORY = None nStart = n for ySelection in ySelectionList: n = nStart if JUST_KEYS: entryName = goodEntryNames[int(scan.split(".")[-1])-1] path = entryName + ySelection if mSelection is not None: mpath = entryName + mSelection mdtype = hdf[mpath].dtype if mdtype not in [numpy.float64, numpy.float32]: mdtype = numpy.float64 mDataset = numpy.asarray(hdf[mpath], dtype=mdtype) if xSelection is not None: xpath = entryName + xSelection xDataset = hdf[xpath][()] else: path = scan + ySelection if mSelection is not None: mpath = scan + mSelection mdtype = hdf[mpath].dtype if mdtype not in [numpy.float64, numpy.float32]: mdtype = numpy.float64 mDataset = numpy.asarray(hdf[mpath], dtype=mdtype) if xSelection is not None: xpath = scan + xSelection xDataset = hdf[xpath][()] try: yDataset = hdf[path] tmpShape = yDataset.shape totalBytes = numpy.ones((1,), yDataset.dtype).itemsize for nItems in tmpShape: totalBytes *= nItems # should one be conservative or just try? if (totalBytes/(1024.*1024.)) > (0.4 * physicalMemory): _logger.info("Force dynamic loading of spectra") #read from disk IN_MEMORY = False else: #read the data into memory _logger.info("Attempt to load whole map into memory") yDataset = hdf[path][()] IN_MEMORY = True except (MemoryError, ValueError): _logger.info("Dynamic loading of spectra") yDataset = hdf[path] IN_MEMORY = False nMcaInYDataset = 1 for dim in yDataset.shape: nMcaInYDataset *= dim nMcaInYDataset = int(nMcaInYDataset/mcaDim) timeData = None if _time is not None: if "live_time" in mcaObjectPaths: # it is assumed that all have the same structure!!! timePath = NexusTools.getMcaObjectPaths(hdf, path)["live_time"] elif "elapsed_time" in mcaObjectPaths: timePath = NexusTools.getMcaObjectPaths(hdf, path)["elapsed_time"] if timePath in hdf: timeData = hdf[timePath][()] elif "::" in timePath: externalFile, externalPath = timePath.split("::") with h5py.File(externalFile, "r") as timeHdf: timeData = timeHdf[externalPath][()] if mcaIndex != 0: if IN_MEMORY: yDataset.shape = -1, mcaDim if mSelection is not None: case = -1 nMonitorData = 1 for v in mDataset.shape: nMonitorData *= v if nMonitorData == nMcaInYDataset: mDataset.shape = nMcaInYDataset case = 0 elif nMonitorData == (nMcaInYDataset * mcaDim): case = 1 mDataset.shape = nMcaInYDataset, mcaDim if case == -1: raise ValueError(\ "I do not know how to handle this monitor data") if timeData is not None: case = -1 nTimeData = 1 for v in timeData.shape: nTimeData *= v if nTimeData == nMcaInYDataset: timeData.shape = nMcaInYDataset case = 0 _time[nStart: nStart + nMcaInYDataset] += timeData if case == -1: _logger.warning("I do not know how to handle this time data") _logger.warning("Ignoring time information") _time= None if (len(yDataset.shape) == 3) and\ (dim1 == yDataset.shape[1]): mca = 0 deltaI = int(yDataset.shape[1]/dim1) for ii in range(yDataset.shape[0]): i = int(n/dim1) yData = yDataset[ii:(ii+1)] yData.shape = -1, mcaDim if mSelection is not None: if case == 0: mData = numpy.outer(mDataset[mca:(mca+dim1)], numpy.ones((mcaDim))) self.data[i, :, :] += yData / mData elif case == 1: mData = mDataset[mca:(mca+dim1), :] mData.shape = -1, mcaDim self.data[i, :, :] += yData / mData else: self.data[i:(i+deltaI), :] += yData n += yDataset.shape[1] mca += dim1 else: for mca in range(nMcaInYDataset): i = int(n/dim1) j = n % dim1 if len(yDataset.shape) == 3: ii = int(mca/yDataset.shape[1]) jj = mca % yDataset.shape[1] yData = yDataset[ii, jj] elif len(yDataset.shape) == 2: yData = yDataset[mca,:] elif len(yDataset.shape) == 1: yData = yDataset if mSelection is not None: if case == 0: self.data[i, j, :] += yData / mDataset[mca] elif case == 1: self.data[i, j, :] += yData / mDataset[mca, :] else: self.data[i, j, :] += yData n += 1 else: if mSelection is not None: case = -1 nMonitorData = 1 for v in mDataset.shape: nMonitorData *= v if nMonitorData == yDataset.shape[0]: case = 3 mDataset.shape = yDataset.shape[0] elif nMonitorData == nMcaInYDataset: mDataset.shape = nMcaInYDataset case = 0 #elif nMonitorData == (yDataset.shape[1] * yDataset.shape[2]): # case = 1 # mDataset.shape = yDataset.shape[1], yDataset.shape[2] if case == -1: raise ValueError(\ "I do not know how to handle this monitor data") if IN_MEMORY: yDataset.shape = mcaDim, -1 if len(yDataset.shape) != 3: for mca in range(nMcaInYDataset): i = int(n/dim1) j = n % dim1 if len(yDataset.shape) == 3: ii = int(mca/yDataset.shape[2]) jj = mca % yDataset.shape[2] yData = yDataset[:, ii, jj] elif len(yDataset.shape) == 2: yData = yDataset[:, mca] elif len(yDataset.shape) == 1: yData = yDataset[:] if mSelection is not None: if case == 0: self.data[i, j, :] += yData / mDataset[mca] elif case == 1: self.data[i, j, :] += yData / mDataset[:, mca] elif case == 3: self.data[i, j, :] += yData / mDataset else: self.data[i, j, :] += yData n += 1 else: #stack of images to be read as MCA for nImage in range(yDataset.shape[0]): tmp = yDataset[nImage:(nImage+1)] if len(tmp.shape) == 3: i = int(n/dim1) j = n % dim1 if 0: #this loop is extremely SLOW!!!(and useless) for ii in range(tmp.shape[1]): for jj in range(tmp.shape[2]): self.data[i+ii, j+jj, nImage] += tmp[0, ii, jj] else: self.data[i:i+tmp.shape[1], j:j+tmp.shape[2], nImage] += tmp[0] if mSelection is not None: for mca in range(yDataset.shape[0]): i = int(n/dim1) j = n % dim1 yData = self.data[i, j, :] if case == 0: self.data[i, j, :] += yData / mDataset[mca] elif case == 1: self.data[i, j, :] += yData / mDataset[:, mca] n += 1 else: n += tmp.shape[1] * tmp.shape[2] yDataset = None if dim0 == 1: self.onProgress(j) if dim0 != 1: self.onProgress(i) self.onEnd() elif not DONE: # data into memory but as images self.info["McaIndex"] = mcaIndex for hdf in hdfStack._sourceObjectList: entryNames = list(hdf["/"].keys()) for scan in scanlist: for ySelection in ySelectionList: if JUST_KEYS: entryName = entryNames[int(scan.split(".")[-1])-1] path = entryName + ySelection if mSelection is not None: mpath = entryName + mSelection mDataset.shape if xSelection is not None: xpath = entryName + xSelection xDataset = hdf[xpath][()] else: path = scan + ySelection if mSelection is not None: mpath = scan + mSelection mdtype = hdf[mpath].dtype if mdtype not in [numpy.float64, numpy.float32]: mdtype = numpy.float64 mDataset = numpy.asarray(hdf[mpath], dtype=mdtype) if xSelection is not None: xpath = scan + xSelection xDataset = hdf[xpath][()] if mSelection is not None: nMonitorData = mDataset.size case = -1 yDatasetShape = yDataset.shape if nMonitorData == yDatasetShape[0]: #as many monitor data as images mDataset.shape = yDatasetShape[0] case = 0 elif nMonitorData == (yDatasetShape[1] * yDatasetShape[2]): #as many monitorData as pixels case = 1 mDataset.shape = yDatasetShape[1], yDatasetShape[2] if case == -1: raise ValueError(\ "I do not know how to handle this monitor data") if case == 0: for i in range(yDatasetShape[0]): self.data[i] += yDataset[i][()] / mDataset[i] elif case == 1: for i in range(yDataset.shape[0]): self.data[i] += yDataset[i] / mDataset else: for i in range(yDataset.shape[0]): self.data[i:i+1] += yDataset[i:i+1] else: self.info["McaIndex"] = mcaIndex if _time: nRequiredValues = 1 for i in range(len(self.data.shape)): if i != mcaIndex: nRequiredValues *= self.data.shape[i] if _time.size != nRequiredValues: _logger.warning("I do not know how to interpret the time information") _logger.warning("Ignoring time information") _time = None else: _time.shape = -1 self.info["SourceType"] = SOURCE_TYPE self.info["SourceName"] = filelist self.info["Size"] = 1 self.info["NumberOfFiles"] = 1 if mcaIndex == 0: self.info["FileIndex"] = 1 else: self.info["FileIndex"] = 0 if _calibration is not None: self.info['McaCalib'] = _calibration else: self.info['McaCalib'] = [ 0.0, 1.0, 0.0] shape = self.data.shape for i in range(len(shape)): key = 'Dim_%d' % (i+1,) self.info[key] = shape[i] self.info['Channel0'] = 0 if xSelection is not None: if xDataset.size == shape[self.info['McaIndex']]: self.x = [xDataset.reshape(-1)] else: _logger.warning("Ignoring xSelection") elif _channels is not None: _channels.shape = -1 self.x = [_channels] if _time is not None: self.info["McaLiveTime"] = _time
def hdf5Slot(self, ddict): entryName = NexusTools.getEntryName(ddict['name']) currentEntry = "%s::%s" % (ddict['file'], entryName) if (currentEntry != self._lastEntry) and not self._BUTTONS: self._lastEntry = None cntList = [] mcaList = [] aliasList = [] measurement = None scanned = [] mcaList = [] if posixpath.dirname(entryName) != entryName: with h5py.File(ddict['file'], "r") as h5file: measurement = NexusTools.getMeasurementGroup(h5file, ddict['name']) scanned = NexusTools.getScannedPositioners(h5file, ddict['name']) if measurement is not None: measurement = [item.name for key,item in measurement.items() \ if self._isNumeric(item)] if self._mca: mcaList = NexusTools.getMcaList(h5file, entryName) for i in range(len(scanned)): key = scanned[i] cntList.append(key) aliasList.append(posixpath.basename(key)) if measurement is not None: for key in measurement: if key not in cntList: cntList.append(key) aliasList.append(posixpath.basename(key)) cleanedCntList = [] for key in cntList: root = key.split('/') root = "/" + root[1] if len(key) == len(root): cleanedCntList.append(key) else: cleanedCntList.append(key[len(root):]) self._autoAliasList = aliasList self._autoCntList = cleanedCntList self.autoTable.build(self._autoCntList, self._autoAliasList) currentTab = qt.safe_str(self.tableTab.tabText( \ self.tableTab.currentIndex())) if self._mca: mcaAliasList = [] cleanedMcaList = [] for key in mcaList: root = key.split('/') root = "/" + root[1] if len(key) == len(root): cleanedMcaList.append(key) else: cleanedMcaList.append(key[len(root):]) mcaAliasList.append(posixpath.basename(key)) self.mcaTable.build(cleanedMcaList, mcaAliasList) nTabs = self.tableTab.count() if (len(mcaList) > 0) and (nTabs < 3): self.tableTab.insertTab(2, self.mcaTable, "MCA") elif (len(mcaList)==0) and (nTabs > 2): self.tableTab.removeTab(2) _logger.debug("currentTab = %s", currentTab) if currentTab != "USER": if (len(mcaList) > 0) and (len(cntList) == 0): idx = self.tableTabOrder.index("MCA") self.tableTab.setCurrentIndex(idx) _logger.debug("setting tab = %s MCA", idx) elif (len(mcaList) == 0) and (len(cntList) > 0): idx = self.tableTabOrder.index("AUTO") self.tableTab.setCurrentIndex(idx) _logger.debug("setting tab = %s AUTO", idx) self._lastEntry = currentEntry if ddict['event'] == 'itemClicked': if ddict['mouse'] == "right": return self.itemRightClickedSlot(ddict) if ddict['mouse'] == "left": # If parent is root do it even if not NXentry?? if ddict['type'] in ['NXentry', 'Entry']: if not self._BUTTONS: auto = self.actions.getConfiguration()["auto"] if auto == "ADD": self._addAction() elif auto == "REPLACE": self._replaceAction() if ddict['event'] == "itemDoubleClicked": if ddict['type'] in ['Dataset']: currentIndex = self.tableTab.currentIndex() text = safe_str(self.tableTab.tabText(currentIndex)) if text.upper() != "USER": if currentIndex == 0: self.tableTab.setCurrentIndex(1) else: self.tableTab.setCurrentIndex(0) if not self._isNumericType(ddict['dtype']): _logger.debug("string like %s", ddict['dtype']) else: root = ddict['name'].split('/') root = "/" + root[1] if len(ddict['name']) == len(root): cnt = ddict['name'] else: cnt = ddict['name'][len(root):] if cnt not in self._cntList: self._cntList.append(cnt) basename = posixpath.basename(cnt) if basename not in self._aliasList: self._aliasList.append(basename) else: self._aliasList.append(cnt) self.cntTable.build(self._cntList, self._aliasList) elif (ddict['color'] == qt.Qt.blue) and ("silx" in sys.modules): # there is an action to be applied self.showInfoWidget(ddict["file"], ddict["name"], dset=False) elif ddict['type'] in ['NXentry', 'Entry']: if self._lastAction is None: return action, selectionType = self._lastAction.split() if action == 'REMOVE': action = 'ADD' ddict['action'] = "%s %s" % (action, selectionType) self.buttonsSlot(ddict) else: if self.data is not None: name = ddict['name'] filename = ddict['file'] fileIndex = self.data.sourceName.index(filename) phynxFile = self.data._sourceObjectList[fileIndex] dataset = phynxFile[name] if isinstance(dataset, h5py.Dataset): root = ddict['name'].split('/') root = "/" + root[1] cnt = ddict['name'].split(root)[-1] if cnt not in self._cntList: _logger.debug("USING SECOND WAY") self._cntList.append(cnt) basename = posixpath.basename(cnt) if basename not in self._aliasList: self._aliasList.append(basename) else: self._aliasList.append(cnt) self.cntTable.build(self._cntList, self._aliasList) return _logger.debug("Unhandled item type: %s", ddict['dtype'])