class DataViz(QtGui.QMainWindow):
"""The main application window for dataviz.
This is an MDI application with dock displaying open HDF5 files on
the left. Attributes of the selected item at left bottom.
Signals
-------
sigOpen: Emitted when a set of files have been selected in the open
files dialog. Sends out the list of file paths selected and the
mode string.
sigCloseFiles: Emitted when the user triggers closeFilesAction. This
is passed on to the HDFTreeWidget which decides which
files to close based on selection.
sigShowAttributes: Emitted when showAttributesAction is
triggered. Connected to
HDFTreeWidget.showAttributes function which
creates a widget for displaying attributes of the
HDF5 node of its current
item. HDFTreeWidget.showAttributes sends a return
signal `attributeWidgetCreated` with the created
widget so that the DataViz widget can incorporate
it as an mdi child window.
sigShowDataset: Emitted when showDatasetAction is
triggered. Connected to HDFTreeWidget's showDataset
function which creates a widget for displaying the
contents of the HDF5 node if it is a dataset.
HDFTreeWidget.showDataset sends a return signal
`sigDatasetWidgetCreated` with the created widget so
that the DataViz widget can incorporate it as an mdi
child window.
sigPlotDataset: Emitted when plotDatasetAction is
triggered. Connected to HDFTreeWidget's plotDataset
function which creates a widget for displaying teh
contents of the HDF5 node if it is a datset.
"""
sigOpen = QtCore.pyqtSignal(list, str)
sigCloseFiles = QtCore.pyqtSignal()
sigShowAttributes = QtCore.pyqtSignal()
sigShowDataset = QtCore.pyqtSignal()
sigPlotDataset = QtCore.pyqtSignal()
def __init__(self, parent=None, flags=QtCore.Qt.WindowFlags(0)):
super(DataViz, self).__init__(parent=parent, flags=flags)
self.readSettings()
self.mdiArea = QtGui.QMdiArea()
self.mdiArea.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
self.mdiArea.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
self.mdiArea.subWindowActivated.connect(self.switchPlotParamPanel)
self.setCentralWidget(self.mdiArea)
self.createTreeDock()
self.createActions()
self.createMenus()
def closeEvent(self, event):
self.writeSettings()
event.accept()
def readSettings(self):
settings = QtCore.QSettings('dataviz', 'dataviz')
self.lastDir = settings.value('lastDir', '.', str)
pos = settings.value('pos', QtCore.QPoint(200, 200))
if isinstance(pos, QtCore.QVariant):
pos = pos.toPyObject()
self.move(pos)
size = settings.value('size', QtCore.QSize(400, 400))
if isinstance(size, QtCore.QVariant):
size = size.toPyObject()
self.resize(size)
def writeSettings(self):
settings = QtCore.QSettings('dataviz', 'dataviz')
settings.setValue('lastDir', self.lastDir)
settings.setValue('pos', self.pos())
settings.setValue('size', self.size())
def openFilesReadOnly(self, filePaths=None):
if filePaths is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read-only', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.filesSelected.connect(self.openFilesReadOnly)
return
# filePaths = QtGui.QFileDialog.getOpenFileNames(self,
# 'Open file(s)', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
filePaths = [str(path)
for path in filePaths] # python2/qt4 compatibility
if len(filePaths) == 0:
return
self.lastDir = QtCore.QFileInfo(filePaths[-1]).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit(filePaths, 'r')
def openFilesReadWrite(self, filePaths=None):
# print(filePaths)
if filePaths is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.filesSelected.connect(self.openFilesReadWrite)
self.fileDialog.show()
return
# filePaths = QtGui.QFileDialog.getOpenFileNames(self,
# 'Open file(s)', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
filePaths = [str(path)
for path in filePaths] # python2/qt4 compatibility
if len(filePaths) == 0:
return
self.lastDir = QtCore.QFileInfo(filePaths[-1]).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit(filePaths, 'r+')
def openFileOverwrite(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.openFileOverwrite)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
self.lastDir = QtCore.QFileInfo(filePath).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit([filePath], 'w')
def createFile(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(
None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.createFile)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
# print('%%%%%', filePath, _)
self.lastDir = filePath.rpartition('/')[0]
# TODO handle recent files
self.sigOpen.emit([filePath], 'w-')
def createActions(self):
self.openFileReadOnlyAction = QtGui.QAction(
QtGui.QIcon(),
'Open file(s) readonly',
self,
shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for reading',
triggered=self.openFilesReadOnly)
self.openFileReadWriteAction = QtGui.QAction(
QtGui.QIcon(),
'&Open file(s) read/write',
self,
# shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for editing',
triggered=self.openFilesReadWrite)
self.openFileOverwriteAction = QtGui.QAction(
QtGui.QIcon(),
'Overwrite file',
self,
# shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for writing (overwrite existing)',
triggered=self.openFileOverwrite)
self.createFileAction = QtGui.QAction(
QtGui.QIcon(),
'&New file',
self,
shortcut=QtGui.QKeySequence.New,
statusTip='Create a new HDF5 file',
triggered=self.createFile)
self.closeFileAction = QtGui.QAction(
QtGui.QIcon(),
'&Close file(s)',
self,
shortcut=QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_K),
statusTip='Close selected files',
triggered=self.sigCloseFiles)
self.quitAction = QtGui.QAction(QtGui.QIcon(),
'&Quit',
self,
shortcut=QtGui.QKeySequence.Quit,
statusTip='Quit dataviz',
triggered=self.doQuit)
self.showAttributesAction = QtGui.QAction(
QtGui.QIcon(),
'Show attributes',
self,
shortcut=QtGui.QKeySequence(QtCore.Qt.Key_Return),
statusTip='Show attributes',
triggered=self.sigShowAttributes)
self.showDatasetAction = QtGui.QAction(
QtGui.QIcon(),
'Show dataset',
self,
shortcut=QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_Return),
statusTip='Show dataset',
triggered=self.sigShowDataset)
self.plotDatasetAction = QtGui.QAction(
QtGui.QIcon(),
'Plot dataset',
self,
shortcut=QtGui.QKeySequence(QtCore.Qt.ALT + QtCore.Qt.Key_P),
statusTip='Plot dataset',
triggered=self.sigPlotDataset)
def createMenus(self):
self.menuBar().setVisible(True)
self.fileMenu = self.menuBar().addMenu('&File')
self.fileMenu.addAction(self.openFileReadWriteAction)
self.fileMenu.addAction(self.openFileReadOnlyAction)
self.fileMenu.addAction(self.openFileOverwriteAction)
self.fileMenu.addAction(self.createFileAction)
self.fileMenu.addAction(self.closeFileAction)
self.fileMenu.addAction(self.quitAction)
self.editMenu = self.menuBar().addMenu('&Edit')
self.editMenu.addAction(self.tree.insertDatasetAction)
self.editMenu.addAction(self.tree.insertGroupAction)
self.editMenu.addAction(self.tree.deleteNodeAction)
self.viewMenu = self.menuBar().addMenu('&View')
self.viewMenu.addAction(self.treeDock.toggleViewAction())
self.dataMenu = self.menuBar().addMenu('&Data')
self.dataMenu.addAction(self.showAttributesAction)
self.dataMenu.addAction(self.showDatasetAction)
self.dataMenu.addAction(self.plotDatasetAction)
def createTreeDock(self):
self.treeDock = QtGui.QDockWidget('File tree', self)
self.tree = HDFTreeWidget(parent=self.treeDock)
self.sigOpen.connect(self.tree.openFiles)
self.tree.doubleClicked.connect(self.tree.createDatasetWidget)
self.tree.sigDatasetWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigDatasetWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigAttributeWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigAttributeWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigPlotWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigPlotWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigPlotParamTreeCreated.connect(self.addPanelBelow)
self.tree.sigDataWidgetActivated.connect(self.activateDataWindow)
# pipe signals of dataviz to those of hdftree widget
self.sigShowAttributes.connect(self.tree.showAttributes)
self.sigShowDataset.connect(self.tree.showDataset)
self.sigPlotDataset.connect(self.tree.plotDataset)
self.sigCloseFiles.connect(self.tree.closeFiles)
self.treeDock.setWidget(self.tree)
self.addDockWidget(QtCore.Qt.LeftDockWidgetArea, self.treeDock)
def addMdiChildWindow(self, widget):
if widget is not None:
subwin = self.mdiArea.addSubWindow(widget)
subwin.setWindowTitle(widget.name)
widget.show()
return subwin
def activateDataWindow(self, widget):
if widget is not None:
for window in self.mdiArea.subWindowList():
if window.widget() == widget:
self.mdiArea.setActiveSubWindow(window)
def closeMdiChildWindow(self, widget):
if widget is not None:
self.tree.removeBufferedWidget(widget)
for window in self.mdiArea.subWindowList():
if window.widget() == widget:
window.deleteLater()
def addPanelBelow(self, widget):
dockWidget = QtGui.QDockWidget(widget.name)
dockWidget.setWidget(widget)
self.addDockWidget(QtCore.Qt.BottomDockWidgetArea, dockWidget)
dockWidget.show()
def switchPlotParamPanel(self, subwin):
"""Make plot param tree panel visible if active subwindow has a
plotwidget. All other plot param trees will be invisible. Qt
does not provide out-of-focus signal for mdi subwindows. So
there is no counterpart of subWindowActivated that can allow
us to hide paramtrees for inactive plot widgets. Hence this
approach.
"""
if subwin is None:
return
for dockWidget in self.findChildren(QtGui.QDockWidget):
# All dockwidgets that contain paramtrees must be checked
if isinstance(dockWidget.widget(), DatasetPlotParamTree):
if isinstance(subwin.widget(), DatasetPlot) and \
dockWidget.widget() in subwin.widget().paramsToPlots:
dockWidget.setVisible(True)
else:
dockWidget.setVisible(False)
def doQuit(self):
self.writeSettings()
QtGui.QApplication.instance().closeAllWindows()
class LibraryEditor(QtWidgets.QWidget):
sigApplyClicked = QtCore.Signal()
sigReloadClicked = QtCore.Signal(object)
def __init__(self, ctrlWidget, library):
super().__init__()
self.setWindowTitle("Manage Library")
self.modules = {} # {mod : [nodes]}
self.paths = set()
self.ctrl = ctrlWidget
self.library = library
self.layout = QtWidgets.QGridLayout(self)
self.loadBtn = QtWidgets.QPushButton("Load Modules", parent=self)
self.loadBtn.clicked.connect(self.loadFile)
# self.reloadBtn = QtWidgets.QPushButton("Reload Selected Modules", parent=self)
# self.reloadBtn.clicked.connect(self.reloadFile)
self.tree = QtWidgets.QTreeWidget(parent=self)
self.tree.setHeaderHidden(True)
self.applyBtn = QtWidgets.QPushButton("Apply", parent=self)
self.applyBtn.clicked.connect(self.applyClicked)
self.layout.addWidget(self.loadBtn, 1, 1, 1, -1)
# self.layout.addWidget(self.reloadBtn, 1, 2, 1, 1)
self.layout.addWidget(self.tree, 2, 1, 1, -1)
self.layout.addWidget(self.applyBtn, 3, 1, 1, -1)
def loadFile(self):
file_filters = "*.py"
self.fileDialog = FileDialog(None, "Load Nodes", None, file_filters)
self.fileDialog.setFileMode(FileDialog.ExistingFiles)
self.fileDialog.filesSelected.connect(self.fileDialogFilesSelected)
self.fileDialog.show()
def fileDialogFilesSelected(self, pths):
dirs = set(map(os.path.dirname, pths))
for pth in dirs:
if pth not in sys.path:
sys.path.append(pth)
self.paths.update(pths)
for mod in pths:
mod = os.path.basename(mod)
mod = os.path.splitext(mod)[0]
mod = importlib.import_module(mod)
if mod in self.modules:
continue
nodes = [getattr(mod, name) for name in dir(mod) if isNodeClass(getattr(mod, name))]
if not nodes:
continue
self.modules[mod] = nodes
parent = QtWidgets.QTreeWidgetItem(self.tree, [mod.__name__])
parent.mod = mod
for node in nodes:
child = QtWidgets.QTreeWidgetItem(parent, [node.__name__])
child.mod = mod
self.tree.expandAll()
def reloadFile(self):
mods = set()
for item in self.tree.selectedItems():
mods.add(item.mod)
for mod in mods:
pg.reload.reload(mod)
self.sigReloadClicked.emit(mods)
def applyClicked(self):
loaded = False
for mod, nodes in self.modules.items():
for node in nodes:
try:
self.library.addNodeType(node, [(mod.__name__, )])
loaded = True
except Exception as e:
printExc(e)
if not loaded:
return
self.ctrl.ui.clear_model(self.ctrl.ui.node_tree)
self.ctrl.ui.create_model(self.ctrl.ui.node_tree, self.library.getLabelTree(rebuild=True))
self.sigApplyClicked.emit()
def saveState(self):
return {'paths': list(self.paths)}
def restoreState(self, state):
self.fileDialogFilesSelected(state['paths'])
class DataViz(QtGui.QMainWindow):
"""The main application window for dataviz.
This is an MDI application with dock displaying open HDF5 files on
the left. Attributes of the selected item at left bottom.
Signals
-------
sigOpen: Emitted when a set of files have been selected in the open
files dialog. Sends out the list of file paths selected and the
mode string.
sigCloseFiles: Emitted when the user triggers closeFilesAction. This
is passed on to the HDFTreeWidget which decides which
files to close based on selection.
sigShowAttributes: Emitted when showAttributesAction is
triggered. Connected to
HDFTreeWidget.showAttributes function which
creates a widget for displaying attributes of the
HDF5 node of its current
item. HDFTreeWidget.showAttributes sends a return
signal `attributeWidgetCreated` with the created
widget so that the DataViz widget can incorporate
it as an mdi child window.
sigShowDataset: Emitted when showDatasetAction is
triggered. Connected to HDFTreeWidget's showDataset
function which creates a widget for displaying the
contents of the HDF5 node if it is a dataset.
HDFTreeWidget.showDataset sends a return signal
`sigDatasetWidgetCreated` with the created widget so
that the DataViz widget can incorporate it as an mdi
child window.
sigPlotDataset: Emitted when plotDatasetAction is
triggered. Connected to HDFTreeWidget's plotDataset
function which creates a widget for displaying teh
contents of the HDF5 node if it is a datset.
"""
sigOpen = QtCore.pyqtSignal(list, str)
sigCloseFiles = QtCore.pyqtSignal()
sigShowAttributes = QtCore.pyqtSignal()
sigShowDataset = QtCore.pyqtSignal()
sigPlotDataset = QtCore.pyqtSignal()
def __init__(self, parent=None, flags=QtCore.Qt.WindowFlags(0)):
super(DataViz, self).__init__(parent=parent, flags=flags)
self.readSettings()
self.mdiArea = QtGui.QMdiArea()
self.mdiArea.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
self.mdiArea.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded)
self.mdiArea.subWindowActivated.connect(self.switchPlotParamPanel)
self.setCentralWidget(self.mdiArea)
self.createTreeDock()
self.createActions()
self.createMenus()
def closeEvent(self, event):
self.writeSettings()
event.accept()
def readSettings(self):
settings = QtCore.QSettings('dataviz', 'dataviz')
self.lastDir = settings.value('lastDir', '.', str)
pos = settings.value('pos', QtCore.QPoint(200, 200))
if isinstance(pos, QtCore.QVariant):
pos = pos.toPyObject()
self.move(pos)
size = settings.value('size', QtCore.QSize(400, 400))
if isinstance(size, QtCore.QVariant):
size = size.toPyObject()
self.resize(size)
def writeSettings(self):
settings = QtCore.QSettings('dataviz', 'dataviz')
settings.setValue('lastDir', self.lastDir)
settings.setValue('pos', self.pos())
settings.setValue('size', self.size())
def openFilesReadOnly(self, filePaths=None):
if filePaths is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read-only', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.filesSelected.connect(self.openFilesReadOnly)
return
# filePaths = QtGui.QFileDialog.getOpenFileNames(self,
# 'Open file(s)', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
filePaths = [str(path) for path in filePaths] # python2/qt4 compatibility
if len(filePaths) == 0:
return
self.lastDir = QtCore.QFileInfo(filePaths[-1]).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit(filePaths, 'r')
def openFilesReadWrite(self, filePaths=None):
# print(filePaths)
if filePaths is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.filesSelected.connect(self.openFilesReadWrite)
self.fileDialog.show()
return
# filePaths = QtGui.QFileDialog.getOpenFileNames(self,
# 'Open file(s)', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
filePaths = [str(path) for path in filePaths] # python2/qt4 compatibility
if len(filePaths) == 0:
return
self.lastDir = QtCore.QFileInfo(filePaths[-1]).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit(filePaths, 'r+')
def openFileOverwrite(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.openFileOverwrite)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
self.lastDir = QtCore.QFileInfo(filePath).dir().absolutePath()
# TODO handle recent files
self.sigOpen.emit([filePath], 'w')
def createFile(self, filePath=None, startDir=None):
if filePath is None or filePaths is False:
self.fileDialog = FileDialog(None, 'Open file(s) read/write', self.lastDir,
'HDF5 file (*.h5 *.hdf);;All files (*)')
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.createFile)
return
# filePath = QtGui.QFileDialog.getOpenFileName(self,
# 'Overwrite file', self.lastDir,
# 'HDF5 file (*.h5 *.hdf);;All files (*)')
if len(filePath) == 0:
return
# print('%%%%%', filePath, _)
self.lastDir = filePath.rpartition('/')[0]
# TODO handle recent files
self.sigOpen.emit([filePath], 'w-')
def createActions(self):
self.openFileReadOnlyAction = QtGui.QAction(QtGui.QIcon(), 'Open file(s) readonly', self,
# shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for reading',
triggered=self.openFilesReadOnly)
self.openFileReadWriteAction = QtGui.QAction(QtGui.QIcon(), '&Open file(s) read/write', self,
shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for editing',
triggered=self.openFilesReadWrite)
self.openFileOverwriteAction = QtGui.QAction(QtGui.QIcon(), 'Overwrite file', self,
# shortcut=QtGui.QKeySequence.Open,
statusTip='Open an HDF5 file for writing (overwrite existing)',
triggered=self.openFileOverwrite)
self.createFileAction = QtGui.QAction(QtGui.QIcon(), '&New file', self,
shortcut=QtGui.QKeySequence.New,
statusTip='Create a new HDF5 file',
triggered=self.createFile)
self.closeFileAction = QtGui.QAction(QtGui.QIcon(), '&Close file(s)',
self,
shortcut=QtGui.QKeySequence(QtCore.Qt.CTRL+QtCore.Qt.Key_K),
statusTip='Close selected files',
triggered=self.sigCloseFiles)
self.quitAction = QtGui.QAction(QtGui.QIcon(), '&Quit', self,
shortcut=QtGui.QKeySequence.Quit,
statusTip='Quit dataviz',
triggered=self.doQuit)
self.showAttributesAction = QtGui.QAction(QtGui.QIcon(), 'Show attributes', self,
shortcut=QtGui.QKeySequence.InsertParagraphSeparator,
statusTip='Show attributes',
triggered=self.sigShowAttributes)
self.showDatasetAction = QtGui.QAction(QtGui.QIcon(), 'Show dataset', self,
shortcut=QtGui.QKeySequence(QtCore.Qt.CTRL+QtCore.Qt.Key_Return),
statusTip='Show dataset',
triggered=self.sigShowDataset)
self.plotDatasetAction = QtGui.QAction(QtGui.QIcon(), 'Plot dataset', self,
shortcut=QtGui.QKeySequence(QtCore.Qt.ALT + QtCore.Qt.Key_P),
statusTip='Plot dataset',
triggered=self.sigPlotDataset)
def createMenus(self):
self.menuBar().setVisible(True)
self.fileMenu = self.menuBar().addMenu('&File')
self.fileMenu.addAction(self.openFileReadWriteAction)
self.fileMenu.addAction(self.openFileReadOnlyAction)
self.fileMenu.addAction(self.openFileOverwriteAction)
self.fileMenu.addAction(self.createFileAction)
self.fileMenu.addAction(self.closeFileAction)
self.fileMenu.addAction(self.quitAction)
self.editMenu = self.menuBar().addMenu('&Edit')
self.editMenu.addAction(self.tree.insertDatasetAction)
self.editMenu.addAction(self.tree.insertGroupAction)
self.editMenu.addAction(self.tree.deleteNodeAction)
self.viewMenu = self.menuBar().addMenu('&View')
self.viewMenu.addAction(self.treeDock.toggleViewAction())
self.dataMenu = self.menuBar().addMenu('&Data')
self.dataMenu.addAction(self.showAttributesAction)
self.dataMenu.addAction(self.showDatasetAction)
self.dataMenu.addAction(self.plotDatasetAction)
def createTreeDock(self):
self.treeDock = QtGui.QDockWidget('File tree', self)
self.tree = HDFTreeWidget(parent=self.treeDock)
self.sigOpen.connect(self.tree.openFiles)
self.tree.doubleClicked.connect(self.tree.createDatasetWidget)
self.tree.sigDatasetWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigDatasetWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigAttributeWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigAttributeWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigPlotWidgetCreated.connect(self.addMdiChildWindow)
self.tree.sigPlotWidgetClosed.connect(self.closeMdiChildWindow)
self.tree.sigPlotParamTreeCreated.connect(self.addPanelBelow)
# pipe signals of dataviz to those of hdftree widget
self.sigShowAttributes.connect(self.tree.showAttributes)
self.sigShowDataset.connect(self.tree.showDataset)
self.sigPlotDataset.connect(self.tree.plotDataset)
self.sigCloseFiles.connect(self.tree.closeFiles)
self.treeDock.setWidget(self.tree)
self.addDockWidget(QtCore.Qt.LeftDockWidgetArea, self.treeDock)
def addMdiChildWindow(self, widget):
if widget is not None:
subwin = self.mdiArea.addSubWindow(widget)
subwin.setWindowTitle(widget.name)
widget.show()
return subwin
def closeMdiChildWindow(self, widget):
if widget is not None:
for window in self.mdiArea.subWindowList():
if window.widget() == widget:
window.deleteLater()
def addPanelBelow(self, widget):
dockWidget = QtGui.QDockWidget(widget.name)
dockWidget.setWidget(widget)
self.addDockWidget(QtCore.Qt.BottomDockWidgetArea, dockWidget)
dockWidget.show()
def switchPlotParamPanel(self, subwin):
"""Make plot param tree panel visible if active subwindow has a
plotwidget. All other plot param trees will be invisible. Qt
does not provide out-of-focus signal for mdi subwindows. So
there is no counterpart of subWindowActivated that can allow
us to hide paramtrees for inactive plot widgets. Hence this
approach.
"""
if subwin is None:
return
for dockWidget in self.findChildren(QtGui.QDockWidget):
# All dockwidgets that contain paramtrees must be checked
if isinstance(dockWidget.widget(), DatasetPlotParamTree):
if isinstance(subwin.widget(), DatasetPlot) and \
dockWidget.widget() in subwin.widget().paramsToPlots:
dockWidget.setVisible(True)
else:
dockWidget.setVisible(False)
def doQuit(self):
self.writeSettings()
QtGui.QApplication.instance().closeAllWindows()
class LogWidget(QtGui.QWidget):
"""A widget to show log entries and filter them.
"""
sigDisplayEntry = QtCore.Signal(object) ## for thread-safetyness
sigAddEntry = QtCore.Signal(object) ## for thread-safetyness
sigScrollToAnchor = QtCore.Signal(object) # for internal use.
Stylesheet = """
body {color: #000; font-family: sans;}
.entry {}
.error .message {color: #900}
.warning .message {color: #740}
.user .message {color: #009}
.status .message {color: #090}
.logExtra {margin-left: 40px;}
.traceback {color: #555; height: 0px;}
.timestamp {color: #000;}
"""
pageTemplate = """
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<style type="text/css">
%s </style>
<script type="text/javascript">
function showDiv(id) {
div = document.getElementById(id);
div.style.visibility = "visible";
div.style.height = "auto";
}
</script>
</head>
<body>
</body>
</html> """ % Stylesheet
def __init__(self, parent):
"""Creates the log widget.
@param object parent: Qt parent object for log widet
"""
QtGui.QWidget.__init__(self, parent)
self.ui = LogWidgetTemplate.Ui_Form()
self.ui.setupUi(self)
#self.ui.input.hide()
self.ui.filterTree.topLevelItem(1).setExpanded(True)
self.entries = [] ## stores all log entries in memory
self.cache = {
} ## for storing html strings of entries that have already been processed
self.displayedEntries = []
self.typeFilters = []
self.importanceFilter = 0
self.dirFilter = False
self.entryArrayBuffer = np.zeros(
1000,
dtype=[ ### a record array for quick filtering of entries
('index', 'int32'), ('importance', 'int32'),
('msgType', '|S10'), ('directory', '|S100'),
('entryId', 'int32')
])
self.entryArray = self.entryArrayBuffer[:0]
self.filtersChanged()
self.sigDisplayEntry.connect(self.displayEntry,
QtCore.Qt.QueuedConnection)
self.sigAddEntry.connect(self.addEntry, QtCore.Qt.QueuedConnection)
self.ui.exportHtmlBtn.clicked.connect(self.exportHtml)
self.ui.filterTree.itemChanged.connect(self.setCheckStates)
self.ui.importanceSlider.valueChanged.connect(self.filtersChanged)
self.ui.output.anchorClicked.connect(self.linkClicked)
self.sigScrollToAnchor.connect(self.scrollToAnchor,
QtCore.Qt.QueuedConnection)
def loadFile(self, f):
"""Load a log file for display.
@param str f: path to file that should be laoded.
f must be able to be read by pyqtgraph configfile.py
"""
log = configfile.readConfigFile(f)
self.entries = []
self.entryArrayBuffer = np.zeros(len(log),
dtype=[('index', 'int32'),
('importance', 'int32'),
('msgType', '|S10'),
('directory', '|S100'),
('entryId', 'int32')])
self.entryArray = self.entryArrayBuffer[:]
i = 0
for k, v in log.items():
v['id'] = k[
9:] ## record unique ID to facilitate HTML generation (javascript needs this ID)
self.entries.append(v)
self.entryArray[i] = np.array(
[(i, v.get('importance', 5), v.get('msgType', 'status'),
v.get('currentDir', ''), v.get('entryId', v['id']))],
dtype=[('index', 'int32'), ('importance', 'int32'),
('msgType', '|S10'), ('directory', '|S100'),
('entryId', 'int32')])
i += 1
self.filterEntries(
) ## puts all entries through current filters and displays the ones that pass
def addEntry(self, entry):
"""Add a log entry to the list.
"""
## All incoming messages begin here
## for thread-safetyness:
isGuiThread = QtCore.QThread.currentThread(
) == QtCore.QCoreApplication.instance().thread()
if not isGuiThread:
self.sigAddEntry.emit(entry)
return
self.entries.append(entry)
i = len(self.entryArray)
entryDir = entry.get('currentDir', None)
if entryDir is None:
entryDir = ''
arr = np.array([
(i, entry['importance'], entry['msgType'], entryDir, entry['id'])
],
dtype=[('index', 'int32'), ('importance', 'int32'),
('msgType', '|S10'), ('directory', '|S100'),
('entryId', 'int32')])
## make more room if needed
if len(self.entryArrayBuffer) == len(self.entryArray):
newArray = np.empty(
len(self.entryArrayBuffer) + 1000, self.entryArrayBuffer.dtype)
newArray[:len(self.entryArray)] = self.entryArray
self.entryArrayBuffer = newArray
self.entryArray = self.entryArrayBuffer[:len(self.entryArray) + 1]
self.entryArray[i] = arr
self.checkDisplay(
entry) ## displays the entry if it passes the current filters
def setCheckStates(self, item, column):
if item == self.ui.filterTree.topLevelItem(1):
if item.checkState(0):
for i in range(item.childCount()):
item.child(i).setCheckState(0, QtCore.Qt.Checked)
elif item.parent() == self.ui.filterTree.topLevelItem(1):
if not item.checkState(0):
self.ui.filterTree.topLevelItem(1).setCheckState(
0, QtCore.Qt.Unchecked)
self.filtersChanged()
def filtersChanged(self):
### Update self.typeFilters, self.importanceFilter, and self.dirFilter to reflect changes.
tree = self.ui.filterTree
self.typeFilters = []
for i in range(tree.topLevelItem(1).childCount()):
child = tree.topLevelItem(1).child(i)
if tree.topLevelItem(1).checkState(0) or child.checkState(0):
text = child.text(0)
self.typeFilters.append(str(text))
self.importanceFilter = self.ui.importanceSlider.value()
# self.updateDirFilter()
self.filterEntries()
# def updateDirFilter(self, dh=None):
# if self.ui.filterTree.topLevelItem(0).checkState(0):
# if dh==None:
# self.dirFilter = self.manager.getDirOfSelectedFile().name()
# else:
# self.dirFilter = dh.name()
# else:
# self.dirFilter = False
def filterEntries(self):
"""Runs each entry in self.entries through the filters and displays if it makes it through."""
### make self.entries a record array, then filtering will be much faster (to OR true/false arrays, + them)
typeMask = self.entryArray['msgType'] == b''
for t in self.typeFilters:
typeMask += self.entryArray['msgType'] == t.encode('ascii')
mask = (self.entryArray['importance'] >
self.importanceFilter) * typeMask
#if self.dirFilter != False:
# d = np.ascontiguousarray(self.entryArray['directory'])
# j = len(self.dirFilter)
# i = len(d)
# d = d.view(np.byte).reshape(i, 100)[:, :j]
# d = d.reshape(i*j).view('|S%d' % j)
# mask *= (d == self.dirFilter)
self.ui.output.clear()
self.ui.output.document().setDefaultStyleSheet(self.Stylesheet)
indices = list(self.entryArray[mask]['index'])
self.displayEntry([self.entries[i] for i in indices])
def checkDisplay(self, entry):
### checks whether entry passes the current filters and displays it if it does.
if entry['msgType'] not in self.typeFilters:
return
elif entry['importance'] < self.importanceFilter:
return
elif self.dirFilter is not False:
if entry['currentDir'][:len(self.dirFilter)] != self.dirFilter:
return
else:
self.displayEntry([entry])
def displayEntry(self, entries):
## entries should be a list of log entries
## for thread-safetyness:
isGuiThread = QtCore.QThread.currentThread(
) == QtCore.QCoreApplication.instance().thread()
if not isGuiThread:
self.sigDisplayEntry.emit(entries)
return
for entry in entries:
if id(entry) not in self.cache:
self.cache[id(entry)] = self.generateEntryHtml(entry)
html = self.cache[id(entry)]
sb = self.ui.output.verticalScrollBar()
isMax = sb.value() == sb.maximum()
self.ui.output.append(html)
self.displayedEntries.append(entry)
if isMax:
## can't scroll to end until the web frame has processed the html change
#frame.setScrollBarValue(QtCore.Qt.Vertical, frame.scrollBarMaximum(QtCore.Qt.Vertical))
## Calling processEvents anywhere inside an error handler is forbidden
## because this can lead to Qt complaining about paint() recursion.
self.sigScrollToAnchor.emit(str(
entry['id'])) ## queued connection
def scrollToAnchor(self, anchor):
self.ui.output.scrollToAnchor(anchor)
def generateEntryHtml(self, entry):
msg = self.cleanText(entry['message'])
reasons = ""
docs = ""
exc = ""
if 'reasons' in entry:
reasons = self.formatReasonStrForHTML(entry['reasons'])
if 'docs' in entry:
docs = self.formatDocsStrForHTML(entry['docs'])
if entry.get('exception', None) is not None:
exc = self.formatExceptionForHTML(entry, entryId=entry['id'])
extra = reasons + docs + exc
if extra != "":
#extra = "<div class='logExtra'>" + extra + "</div>"
extra = "<table class='logExtra'><tr><td>" + extra + "</td></tr></table>"
#return """
#<div class='entry'>
#<div class='%s'>
#<span class='timestamp'>%s</span>
#<span class='message'>%s</span>
#%s
#</div>
#</div>
#""" % (entry['msgType'], entry['timestamp'], msg, extra)
return """
<a name="%s"/><table class='entry'><tr><td>
<table class='%s'><tr><td>
<span class='timestamp'>%s</span>
<span class='message'>%s</span>
%s
</td></tr></table>
</td></tr></table>
""" % (str(
entry['id']), entry['msgType'], entry['timestamp'], msg, extra)
@staticmethod
def cleanText(text):
text = re.sub(r'&', '&', text)
text = re.sub(r'>', '>', text)
text = re.sub(r'<', '<', text)
text = re.sub(r'\n', '<br/>\n', text)
return text
def formatExceptionForHTML(self,
entry,
exception=None,
count=1,
entryId=None):
### Here, exception is a dict that holds the message, reasons, docs, traceback and oldExceptions (which are also dicts, with the same entries)
## the count and tracebacks keywords are for calling recursively
if exception is None:
exception = entry['exception']
#if tracebacks is None:
#tracebacks = []
indent = 10
text = self.cleanText(exception['message'])
text = re.sub(r'^HelpfulException: ', '', text)
messages = [text]
if 'reasons' in exception:
reasons = self.formatReasonsStrForHTML(exception['reasons'])
text += reasons
#self.displayText(reasons, entry, color, clean=False)
if 'docs' in exception:
docs = self.formatDocsStrForHTML(exception['docs'])
#self.displayText(docs, entry, color, clean=False)
text += docs
traceback = [
self.formatTracebackForHTML(exception['traceback'], count)
]
text = [text]
if 'oldExc' in exception:
exc, tb, msgs = self.formatExceptionForHTML(entry,
exception['oldExc'],
count=count + 1)
text.extend(exc)
messages.extend(msgs)
traceback.extend(tb)
#else:
#if len(tracebacks)==count+1:
#n=0
#else:
#n=1
#for i, tb in enumerate(tracebacks):
#self.displayTraceback(tb, entry, number=i+n)
if count == 1:
exc = "<div class=\"exception\"><ol>" + "\n".join(
["<li>%s</li>" % ex for ex in text]) + "</ol></div>"
tbStr = "\n".join([
"<li><b>%s</b><br/><span class='traceback'>%s</span></li>" %
(messages[i], tb) for i, tb in enumerate(traceback)
])
#traceback = "<div class=\"traceback\" id=\"%s\"><ol>"%str(entryId) + tbStr + "</ol></div>"
entry['tracebackHtml'] = tbStr
#return exc + '<a href="#" onclick="showDiv(\'%s\')">Show traceback</a>'%str(entryId) + traceback
return exc + '<a href="exc:%s">Show traceback %s</a>' % (
str(entryId), str(entryId))
else:
return text, traceback, messages
def formatTracebackForHTML(self, tb, number):
try:
tb = [
line for line in tb
if not line.startswith("Traceback (most recent call last)")
]
except:
print("\n" + str(tb) + "\n")
raise
return re.sub(" ", " ", ("").join(map(self.cleanText, tb)))[:-1]
#tb = [self.cleanText(strip(x)) for x in tb]
#lines = []
#prefix = ''
#for l in ''.join(tb).split('\n'):
#if l == '':
#continue
#if l[:9] == "Traceback":
#prefix = ' ' + str(number) + '. '
#continue
#spaceCount = 0
#while l[spaceCount] == ' ':
#spaceCount += 1
#if prefix is not '':
#spaceCount -= 1
#lines.append(" "*(spaceCount*4) + prefix + l)
#prefix = ''
#return '<div class="traceback">' + '<br />'.join(lines) + '</div>'
#self.displayText('<br />'.join(lines), entry, color, clean=False)
def formatReasonsStrForHTML(self, reasons):
#indent = 6
reasonStr = "<table class='reasons'><tr><td>Possible reasons include:\n<ul>\n"
for r in reasons:
r = self.cleanText(r)
reasonStr += "<li>" + r + "</li>\n"
#reasonStr += " "*22 + chr(97+i) + ". " + r + "<br>"
reasonStr += "</ul></td></tr></table>\n"
return reasonStr
def formatDocsStrForHTML(self, docs):
#indent = 6
docStr = "<div class='docRefs'>Relevant documentation:\n<ul>\n"
for d in docs:
d = self.cleanText(d)
docStr += "<li><a href=\"doc:%s\">%s</a></li>\n" % (d, d)
docStr += "</ul></div>\n"
return docStr
def exportHtml(self, fileName=False):
if fileName is False:
self.fileDialog = FileDialog(self, "Save HTML as...",
"htmltemp.log")
#self.fileDialog.setFileMode(QtGui.QFileDialog.AnyFile)
self.fileDialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.exportHtml)
return
if fileName[-5:] != '.html':
fileName += '.html'
#doc = self.ui.output.document().toHtml('utf-8')
#for e in self.displayedEntries:
#if e.has_key('tracebackHtml'):
#doc = re.sub(r'<a href="exc:%s">(<[^>]+>)*Show traceback %s(<[^>]+>)*</a>'%(str(e['id']), str(e['id'])), e['tracebackHtml'], doc)
doc = self.pageTemplate
for e in self.displayedEntries:
doc += self.cache[id(e)]
for e in self.displayedEntries:
if 'tracebackHtml' in e:
doc = re.sub(
r'<a href="exc:%s">(<[^>]+>)*Show traceback %s(<[^>]+>)*</a>'
% (str(e['id']), str(e['id'])), e['tracebackHtml'], doc)
f = open(fileName, 'w')
f.write(doc)
f.close()
def linkClicked(self, url):
url = url.toString()
if url[:4] == 'doc:':
#self.manager.showDocumentation(url[4:])
print("Not implemented")
elif url[:4] == 'exc:':
cursor = self.ui.output.document().find('Show traceback %s' %
url[4:])
try:
tb = self.entries[int(url[4:]) - 1]['tracebackHtml']
except IndexError:
try:
tb = self.entries[self.entryArray[
self.entryArray['entryId'] == (
int(url[4:]))]['index']]['tracebackHtml']
except:
print("requested index %d, but only %d entries exist." %
(int(url[4:]) - 1, len(self.entries)))
raise
cursor.insertHtml(tb)
def clear(self):
self.ui.output.clear()
self.displayedEntryies = []
class SourceConfiguration(QtGui.QWidget):
sigApply = QtCore.Signal(object) # src_cfg dict
def __init__(self, parent=None):
super().__init__(parent)
self.setWindowTitle("Configure")
self.formLayout = QtWidgets.QFormLayout(self)
self.interval = QtGui.QDoubleSpinBox(self)
self.interval.setValue(0.01)
self.formLayout.addRow("Interval", self.interval)
self.init_time = QtGui.QDoubleSpinBox(self)
self.init_time.setValue(0.5)
self.formLayout.addRow("Init Time", self.init_time)
self.hb_period = QtGui.QSpinBox(self)
self.hb_period.setValue(10)
self.formLayout.addRow("Heartbeat Period", self.hb_period)
self.source_type = QtGui.QComboBox(self)
self.source_type.addItem("hdf5")
self.source_type.addItem("psana")
self.formLayout.addRow("Source Type", self.source_type)
self.repeat = QtWidgets.QCheckBox(self)
self.repeat.setChecked(True)
self.formLayout.addRow("Repeat", self.repeat)
self.files = []
self.fileListView = QtWidgets.QListView(self)
self.fileListView.setSelectionMode(
QtWidgets.QAbstractItemView.ExtendedSelection)
self.fileListModel = QtCore.QStringListModel(self.files)
self.fileListView.setModel(self.fileListModel)
self.formLayout.addRow(self.fileListView)
self.horizontalLayout = QtWidgets.QHBoxLayout()
self.addBtn = QtWidgets.QPushButton("Add", parent=self)
self.addBtn.clicked.connect(self.addFile)
self.horizontalLayout.addWidget(self.addBtn)
self.removeBtn = QtWidgets.QPushButton("Remove", parent=self)
self.removeBtn.clicked.connect(self.removeFiles)
self.horizontalLayout.addWidget(self.removeBtn)
self.applyBtn = QtWidgets.QPushButton("Apply", parent=self)
self.applyBtn.clicked.connect(self.applyClicked)
self.horizontalLayout.addWidget(self.applyBtn)
self.formLayout.addRow(self.horizontalLayout)
def addFile(self):
file_filters = self.source_type.currentText()
if file_filters == "hdf5":
file_filters = "*.h5 *.hdf5"
elif file_filters == "psana":
file_filters = "*.xtc2"
self.fileDialog = FileDialog(None, "Load Data", None, file_filters)
self.fileDialog.setFileMode(FileDialog.ExistingFiles)
self.fileDialog.filesSelected.connect(self.fileDialogFilesSelected)
self.fileDialog.show()
def removeFiles(self):
selectionModel = self.fileListView.selectionModel()
for pth in selectionModel.selection().indexes():
pth = pth.data()
self.files.remove(pth)
self.fileListModel.setStringList(self.files)
def fileDialogFilesSelected(self, pths):
self.files.extend(pths)
self.fileListModel.setStringList(self.files)
def saveState(self):
cfg = {}
cfg['type'] = self.source_type.currentText()
cfg['interval'] = self.interval.value()
cfg['init_time'] = self.init_time.value()
cfg['hb_period'] = self.hb_period.value()
cfg['files'] = self.files
cfg['repeat'] = self.repeat.isChecked()
return cfg
def restoreState(self, state):
self.source_type.setCurrentText(state['type'])
self.interval.setValue(state['interval'])
self.init_time.setValue(state['init_time'])
self.hb_period.setValue(state['hb_period'])
self.files = state['files']
self.fileListModel.setStringList(self.files)
self.repeat.setChecked(state['repeat'])
def applyClicked(self):
cfg = self.saveState()
self.sigApply.emit(cfg)
class FlowchartCtrlWidget(QtGui.QWidget):
"""
The widget that contains the list of all the nodes in a flowchart and their controls,
as well as buttons for loading/saving flowcharts.
"""
def __init__(self, chart, graphmgr_addr):
super().__init__()
self.graphCommHandler = AsyncGraphCommHandler(graphmgr_addr.name,
graphmgr_addr.comm,
ctx=chart.ctx)
self.graph_name = graphmgr_addr.name
self.metadata = None
self.currentFileName = None
self.chart = chart
self.chartWidget = FlowchartWidget(chart, self)
self.ui = EditorTemplate.Ui_Toolbar()
self.ui.setupUi(parent=self, chart=self.chartWidget)
self.ui.create_model(self.ui.node_tree,
self.chart.library.getLabelTree())
self.ui.create_model(self.ui.source_tree,
self.chart.source_library.getLabelTree())
self.chart.sigNodeChanged.connect(self.ui.setPending)
self.features = Features(self.graphCommHandler)
self.ui.actionNew.triggered.connect(self.clear)
self.ui.actionOpen.triggered.connect(self.openClicked)
self.ui.actionSave.triggered.connect(self.saveClicked)
self.ui.actionSaveAs.triggered.connect(self.saveAsClicked)
self.ui.actionConfigure.triggered.connect(self.configureClicked)
self.ui.actionApply.triggered.connect(self.applyClicked)
self.ui.actionReset.triggered.connect(self.resetClicked)
# self.ui.actionProfiler.triggered.connect(self.profilerClicked)
self.ui.actionHome.triggered.connect(self.homeClicked)
self.ui.navGroup.triggered.connect(self.navClicked)
self.chart.sigFileLoaded.connect(self.setCurrentFile)
self.chart.sigFileSaved.connect(self.setCurrentFile)
self.sourceConfigure = SourceConfiguration()
self.sourceConfigure.sigApply.connect(self.configureApply)
self.libraryEditor = EditorTemplate.LibraryEditor(self, chart.library)
self.libraryEditor.sigApplyClicked.connect(self.libraryUpdated)
self.libraryEditor.sigReloadClicked.connect(self.libraryReloaded)
self.ui.libraryConfigure.clicked.connect(self.libraryEditor.show)
self.graph_info = pc.Info('ami_graph', 'AMI Client graph',
['hutch', 'name'])
@asyncSlot()
async def applyClicked(self, build_views=True):
graph_nodes = []
disconnectedNodes = []
displays = set()
msg = QtWidgets.QMessageBox(parent=self)
msg.setText("Failed to submit graph! See status.")
if self.chart.deleted_nodes:
await self.graphCommHandler.remove(self.chart.deleted_nodes)
self.chart.deleted_nodes = []
# detect if the manager has no graph (e.g. from a purge on failure)
if await self.graphCommHandler.graphVersion == 0:
# mark all the nodes as changed to force a resubmit of the whole graph
for name, gnode in self.chart._graph.nodes().items():
gnode = gnode['node']
gnode.changed = True
# reset reference counting on views
await self.features.reset()
outputs = [n for n, d in self.chart._graph.out_degree() if d == 0]
changed_nodes = set()
failed_nodes = set()
seen = set()
for name, gnode in self.chart._graph.nodes().items():
gnode = gnode['node']
if not gnode.enabled():
continue
if not gnode.hasInput():
disconnectedNodes.append(gnode)
continue
elif gnode.exception:
gnode.clearException()
gnode.recolor()
if gnode.changed and gnode not in changed_nodes:
changed_nodes.add(gnode)
if not hasattr(gnode, 'to_operation'):
if gnode.isSource() and gnode.viewable() and gnode.viewed:
displays.add(gnode)
elif gnode.exportable():
displays.add(gnode)
continue
for output in outputs:
paths = list(
nx.algorithms.all_simple_paths(self.chart._graph, name,
output))
if not paths:
paths = [[output]]
for path in paths:
for gnode in path:
gnode = self.chart._graph.nodes[gnode]
node = gnode['node']
if hasattr(node,
'to_operation') and node not in seen:
try:
nodes = node.to_operation(
inputs=node.input_vars(),
conditions=node.condition_vars())
except Exception:
self.chartWidget.updateStatus(
f"{node.name()} error!", color='red')
printExc(
f"{node.name()} raised exception! See console for stacktrace."
)
node.setException(True)
failed_nodes.add(node)
continue
seen.add(node)
if type(nodes) is list:
graph_nodes.extend(nodes)
else:
graph_nodes.append(nodes)
if (node.viewable()
or node.buffered()) and node.viewed:
displays.add(node)
if disconnectedNodes:
for node in disconnectedNodes:
self.chartWidget.updateStatus(f"{node.name()} disconnected!",
color='red')
node.setException(True)
msg.exec()
return
if failed_nodes:
self.chartWidget.updateStatus("failed to submit graph",
color='red')
msg.exec()
return
if graph_nodes:
await self.graphCommHandler.add(graph_nodes)
node_names = ', '.join(
set(map(lambda node: node.parent, graph_nodes)))
self.chartWidget.updateStatus(f"Submitted {node_names}")
node_names = ', '.join(set(map(lambda node: node.name(), displays)))
if displays and build_views:
self.chartWidget.updateStatus(f"Redisplaying {node_names}")
await self.chartWidget.build_views(displays,
export=True,
redisplay=True)
for node in changed_nodes:
node.changed = False
self.metadata = await self.graphCommHandler.metadata
self.ui.setPendingClear()
version = str(await self.graphCommHandler.graphVersion)
state = self.chart.saveState()
state = json.dumps(state,
indent=2,
separators=(',', ': '),
sort_keys=True,
cls=amitypes.TypeEncoder)
self.graph_info.labels(self.chart.hutch, self.graph_name).info({
'graph':
state,
'version':
version
})
def openClicked(self):
startDir = self.chart.filePath
if startDir is None:
startDir = '.'
self.fileDialog = FileDialog(None, "Load Flowchart..", startDir,
"Flowchart (*.fc)")
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.chart.loadFile)
def saveClicked(self):
if self.currentFileName is None:
self.saveAsClicked()
else:
try:
self.chart.saveFile(self.currentFileName)
except Exception as e:
raise e
def saveAsClicked(self):
try:
if self.currentFileName is None:
self.chart.saveFile()
else:
self.chart.saveFile(suggestedFileName=self.currentFileName)
except Exception as e:
raise e
def setCurrentFile(self, fileName):
self.currentFileName = fileName
def homeClicked(self):
children = self.viewBox().allChildren()
self.viewBox().autoRange(items=children)
def navClicked(self, action):
if action == self.ui.actionPan:
self.viewBox().setMouseMode("Pan")
elif action == self.ui.actionSelect:
self.viewBox().setMouseMode("Select")
elif action == self.ui.actionComment:
self.viewBox().setMouseMode("Comment")
@asyncSlot()
async def resetClicked(self):
await self.graphCommHandler.destroy()
for name, gnode in self.chart._graph.nodes().items():
gnode = gnode['node']
gnode.changed = True
await self.applyClicked()
def scene(self):
# returns the GraphicsScene object
return self.chartWidget.scene()
def viewBox(self):
return self.chartWidget.viewBox()
def chartWidget(self):
return self.chartWidget
@asyncSlot()
async def clear(self):
await self.graphCommHandler.destroy()
await self.chart.clear()
self.chartWidget.clear()
self.setCurrentFile(None)
self.chart.sigFileLoaded.emit('')
self.features = Features(self.graphCommHandler)
def configureClicked(self):
self.sourceConfigure.show()
@asyncSlot(object)
async def configureApply(self, src_cfg):
missing = []
if 'files' in src_cfg:
for f in src_cfg['files']:
if not os.path.exists(f):
missing.append(f)
if not missing:
await self.graphCommHandler.updateSources(src_cfg)
else:
missing = ' '.join(missing)
self.chartWidget.updateStatus(f"Missing {missing}!", color='red')
@asyncSlot()
async def profilerClicked(self):
await self.chart.broker.send_string("profiler", zmq.SNDMORE)
await self.chart.broker.send_pyobj(
fcMsgs.Profiler(name=self.graph_name, command="show"))
@asyncSlot()
async def libraryUpdated(self):
await self.chart.broker.send_string("library", zmq.SNDMORE)
await self.chart.broker.send_pyobj(
fcMsgs.Library(name=self.graph_name,
paths=self.libraryEditor.paths))
dirs = set(map(os.path.dirname, self.libraryEditor.paths))
await self.graphCommHandler.updatePath(dirs)
self.chartWidget.updateStatus("Loaded modules.")
@asyncSlot(object)
async def libraryReloaded(self, mods):
smods = set(map(lambda mod: mod.__name__, mods))
for name, gnode in self.chart._graph.nodes().items():
node = gnode['node']
if node.__module__ in smods:
await self.chart.broker.send_string(node.name(), zmq.SNDMORE)
await self.chart.broker.send_pyobj(
fcMsgs.ReloadLibrary(name=node.name(), mods=smods))
self.chartWidget.updateStatus(f"Reloaded {node.name()}.")
class Flowchart(Node):
sigFileLoaded = QtCore.Signal(object)
sigFileSaved = QtCore.Signal(object)
sigNodeCreated = QtCore.Signal(object)
sigNodeChanged = QtCore.Signal(object)
# called when output is expected to have changed
def __init__(self,
name=None,
filePath=None,
library=None,
broker_addr="",
graphmgr_addr="",
checkpoint_addr="",
prometheus_dir=None,
hutch=None):
super().__init__(name)
self.socks = []
self.library = library or LIBRARY
self.graphmgr_addr = graphmgr_addr
self.source_library = None
self.ctx = zmq.asyncio.Context()
self.broker = self.ctx.socket(
zmq.PUB) # used to create new node processes
self.broker.connect(broker_addr)
self.socks.append(self.broker)
self.graphinfo = self.ctx.socket(zmq.SUB)
self.graphinfo.setsockopt_string(zmq.SUBSCRIBE, '')
self.graphinfo.connect(graphmgr_addr.info)
self.socks.append(self.graphinfo)
self.checkpoint = self.ctx.socket(
zmq.SUB) # used to receive ctrlnode updates from processes
self.checkpoint.setsockopt_string(zmq.SUBSCRIBE, '')
self.checkpoint.connect(checkpoint_addr)
self.socks.append(self.checkpoint)
self.filePath = filePath
self._graph = nx.MultiDiGraph()
self.nextZVal = 10
self._widget = None
self._scene = None
self.deleted_nodes = []
self.prometheus_dir = prometheus_dir
self.hutch = hutch
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.close()
def close(self):
for sock in self.socks:
sock.close(linger=0)
if self._widget is not None:
self._widget.graphCommHandler.close()
self.ctx.term()
def start_prometheus(self):
port = Ports.Prometheus
while True:
try:
pc.start_http_server(port)
break
except OSError:
port += 1
if self.prometheus_dir:
if not os.path.exists(self.prometheus_dir):
os.makedirs(self.prometheus_dir)
pth = f"drpami_{socket.gethostname()}_client.json"
pth = os.path.join(self.prometheus_dir, pth)
conf = [{"targets": [f"{socket.gethostname()}:{port}"]}]
try:
with open(pth, 'w') as f:
json.dump(conf, f)
except PermissionError:
logging.error("Permission denied: %s", pth)
pass
def setLibrary(self, lib):
self.library = lib
self.widget().chartWidget.buildMenu()
def nodes(self, **kwargs):
return self._graph.nodes(**kwargs)
def createNode(self, nodeType=None, name=None, pos=None):
"""Create a new Node and add it to this flowchart.
"""
if name is None:
n = 0
while True:
name = "%s.%d" % (nodeType, n)
if name not in self._graph.nodes():
break
n += 1
# create an instance of the node
node = self.library.getNodeType(nodeType)(name)
self.addNode(node, pos)
return node
def addNode(self, node, pos=None):
"""Add an existing Node to this flowchart.
See also: createNode()
"""
if pos is None:
pos = [0, 0]
if type(pos) in [QtCore.QPoint, QtCore.QPointF]:
pos = [pos.x(), pos.y()]
item = node.graphicsItem()
item.setZValue(self.nextZVal * 2)
self.nextZVal += 1
self.viewBox.addItem(item)
pos = (find_nearest(pos[0]), find_nearest(pos[1]))
item.moveBy(*pos)
self._graph.add_node(node.name(), node=node)
node.sigClosed.connect(self.nodeClosed)
node.sigTerminalConnected.connect(self.nodeConnected)
node.sigTerminalDisconnected.connect(self.nodeDisconnected)
node.sigNodeEnabled.connect(self.nodeEnabled)
self.sigNodeCreated.emit(node)
if node.isChanged(True, True):
self.sigNodeChanged.emit(node)
@asyncSlot(object, object)
async def nodeClosed(self, node, input_vars):
self._graph.remove_node(node.name())
await self.broker.send_string(node.name(), zmq.SNDMORE)
await self.broker.send_pyobj(fcMsgs.CloseNode())
ctrl = self.widget()
name = node.name()
if hasattr(node, 'to_operation'):
self.deleted_nodes.append(name)
self.sigNodeChanged.emit(node)
elif isinstance(node, SourceNode):
await ctrl.features.discard(name, name)
await ctrl.graphCommHandler.unview(name)
elif node.viewable():
views = []
for term, in_var in input_vars.items():
discarded = await ctrl.features.discard(name, in_var)
if discarded:
views.append(in_var)
if views:
await ctrl.graphCommHandler.unview(views)
def nodeConnected(self, localTerm, remoteTerm):
if remoteTerm.isOutput() or localTerm.isCondition():
t = remoteTerm
remoteTerm = localTerm
localTerm = t
localNode = localTerm.node().name()
remoteNode = remoteTerm.node().name()
key = localNode + '.' + localTerm.name(
) + '->' + remoteNode + '.' + remoteTerm.name()
if not self._graph.has_edge(localNode, remoteNode, key=key):
self._graph.add_edge(localNode,
remoteNode,
key=key,
from_term=localTerm.name(),
to_term=remoteTerm.name())
self.sigNodeChanged.emit(localTerm.node())
def nodeDisconnected(self, localTerm, remoteTerm):
if remoteTerm.isOutput() or localTerm.isCondition():
t = remoteTerm
remoteTerm = localTerm
localTerm = t
localNode = localTerm.node().name()
remoteNode = remoteTerm.node().name()
key = localNode + '.' + localTerm.name(
) + '->' + remoteNode + '.' + remoteTerm.name()
if self._graph.has_edge(localNode, remoteNode, key=key):
self._graph.remove_edge(localNode, remoteNode, key=key)
self.sigNodeChanged.emit(localTerm.node())
@asyncSlot(object)
async def nodeEnabled(self, root):
enabled = root._enabled
outputs = [n for n, d in self._graph.out_degree() if d == 0]
sources_targets = list(it.product([root.name()], outputs))
ctrl = self.widget()
views = []
for s, t in sources_targets:
paths = list(nx.algorithms.all_simple_paths(self._graph, s, t))
for path in paths:
for node in path:
node = self._graph.nodes[node]['node']
name = node.name()
node.nodeEnabled(enabled)
if not enabled:
if hasattr(node, 'to_operation'):
self.deleted_nodes.append(name)
elif node.viewable():
for term, in_var in node.input_vars().items():
discarded = await ctrl.features.discard(
name, in_var)
if discarded:
views.append(in_var)
else:
node.changed = True
if node.conditions():
preds = self._graph.predecessors(node.name())
preds = filter(lambda n: n.startswith("Filter"), preds)
for filt in preds:
node = self._graph.nodes[filt]['node']
node.nodeEnabled(enabled)
if views:
await ctrl.graphCommHandler.unview(views)
await ctrl.applyClicked()
def connectTerminals(self, term1, term2, type_file=None):
"""Connect two terminals together within this flowchart."""
term1.connectTo(term2, type_file=type_file)
def chartGraphicsItem(self):
"""
Return the graphicsItem that displays the internal nodes and
connections of this flowchart.
Note that the similar method `graphicsItem()` is inherited from Node
and returns the *external* graphical representation of this flowchart."""
return self.viewBox
def widget(self):
"""
Return the control widget for this flowchart.
This widget provides GUI access to the parameters for each node and a
graphical representation of the flowchart.
"""
if self._widget is None:
self._widget = FlowchartCtrlWidget(self, self.graphmgr_addr)
self.scene = self._widget.scene()
self.viewBox = self._widget.viewBox()
return self._widget
def saveState(self):
"""
Return a serializable data structure representing the current state of this flowchart.
"""
state = {}
state['nodes'] = []
state['connects'] = []
state['viewbox'] = self.viewBox.saveState()
for name, node in self.nodes(data='node'):
cls = type(node)
clsName = cls.__name__
ns = {'class': clsName, 'name': name, 'state': node.saveState()}
state['nodes'].append(ns)
for from_node, to_node, data in self._graph.edges(data=True):
from_term = data['from_term']
to_term = data['to_term']
state['connects'].append((from_node, from_term, to_node, to_term))
state['source_configuration'] = self.widget(
).sourceConfigure.saveState()
state['library'] = self.widget().libraryEditor.saveState()
return state
def restoreState(self, state):
"""
Restore the state of this flowchart from a previous call to `saveState()`.
"""
self.blockSignals(True)
try:
if 'source_configuration' in state:
src_cfg = state['source_configuration']
self.widget().sourceConfigure.restoreState(src_cfg)
if src_cfg['files']:
self.widget().sourceConfigure.applyClicked()
if 'library' in state:
lib_cfg = state['library']
self.widget().libraryEditor.restoreState(lib_cfg)
self.widget().libraryEditor.applyClicked()
if 'viewbox' in state:
self.viewBox.restoreState(state['viewbox'])
nodes = state['nodes']
nodes.sort(key=lambda a: a['state']['pos'][0])
for n in nodes:
if n['class'] == 'SourceNode':
try:
ttype = eval(n['state']['terminals']['Out']['ttype'])
n['state']['terminals']['Out']['ttype'] = ttype
node = SourceNode(name=n['name'],
terminals=n['state']['terminals'])
self.addNode(node=node)
except Exception:
printExc(
"Error creating node %s: (continuing anyway)" %
n['name'])
else:
try:
node = self.createNode(n['class'], name=n['name'])
except Exception:
printExc(
"Error creating node %s: (continuing anyway)" %
n['name'])
node.restoreState(n['state'])
if hasattr(node, "display"):
node.display(topics=None, terms=None, addr=None, win=None)
if hasattr(node.widget,
'restoreState') and 'widget' in n['state']:
node.widget.restoreState(n['state']['widget'])
connections = {}
with tempfile.NamedTemporaryFile(mode='w') as type_file:
type_file.write("from mypy_extensions import TypedDict\n")
type_file.write("from typing import *\n")
type_file.write("import numbers\n")
type_file.write("import amitypes\n")
type_file.write("T = TypeVar('T')\n\n")
nodes = self.nodes(data='node')
for n1, t1, n2, t2 in state['connects']:
try:
node1 = nodes[n1]
term1 = node1[t1]
node2 = nodes[n2]
term2 = node2[t2]
self.connectTerminals(term1, term2, type_file)
if term1.isInput() or term1.isCondition:
in_name = node1.name() + '_' + term1.name()
in_name = in_name.replace('.', '_')
out_name = node2.name() + '_' + term2.name()
out_name = out_name.replace('.', '_')
else:
in_name = node2.name() + '_' + term2.name()
in_name = in_name.replace('.', '_')
out_name = node1.name() + '_' + term1.name()
out_name = out_name.replace('.', '_')
connections[(in_name, out_name)] = (term1, term2)
except Exception:
print(node1.terminals)
print(node2.terminals)
printExc("Error connecting terminals %s.%s - %s.%s:" %
(n1, t1, n2, t2))
type_file.flush()
status = subprocess.run(
["mypy", "--follow-imports", "silent", type_file.name],
capture_output=True,
text=True)
if status.returncode != 0:
lines = status.stdout.split('\n')[:-1]
for line in lines:
m = re.search(r"\"+(\w+)\"+", line)
if m:
m = m.group().replace('"', '')
for i in connections:
if i[0] == m:
term1, term2 = connections[i]
term1.disconnectFrom(term2)
break
elif i[1] == m:
term1, term2 = connections[i]
term1.disconnectFrom(term2)
break
finally:
self.blockSignals(False)
for name, node in self.nodes(data='node'):
self.sigNodeChanged.emit(node)
@asyncSlot(str)
async def loadFile(self, fileName=None):
"""
Load a flowchart (*.fc) file.
"""
with open(fileName, 'r') as f:
state = json.load(f)
ctrl = self.widget()
await ctrl.clear()
self.restoreState(state)
self.viewBox.autoRange()
self.sigFileLoaded.emit(fileName)
await ctrl.applyClicked(build_views=False)
nodes = []
for name, node in self.nodes(data='node'):
if node.viewed:
nodes.append(node)
await ctrl.chartWidget.build_views(nodes, ctrl=True, export=True)
def saveFile(self,
fileName=None,
startDir=None,
suggestedFileName='flowchart.fc'):
"""
Save this flowchart to a .fc file
"""
if fileName is None:
if startDir is None:
startDir = self.filePath
if startDir is None:
startDir = '.'
self.fileDialog = FileDialog(None, "Save Flowchart..", startDir,
"Flowchart (*.fc)")
self.fileDialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.saveFile)
return
if not fileName.endswith('.fc'):
fileName += ".fc"
state = self.saveState()
state = json.dumps(state,
indent=2,
separators=(',', ': '),
sort_keys=True,
cls=amitypes.TypeEncoder)
with open(fileName, 'w') as f:
f.write(state)
f.write('\n')
ctrl = self.widget()
ctrl.graph_info.labels(self.hutch,
ctrl.graph_name).info({'graph': state})
ctrl.chartWidget.updateStatus(f"Saved graph to: {fileName}")
self.sigFileSaved.emit(fileName)
async def clear(self):
"""
Remove all nodes from this flowchart except the original input/output nodes.
"""
for name, gnode in self._graph.nodes().items():
node = gnode['node']
await self.broker.send_string(name, zmq.SNDMORE)
await self.broker.send_pyobj(fcMsgs.CloseNode())
node.close(emit=False)
self._graph = nx.MultiDiGraph()
async def updateState(self):
while True:
node_name = await self.checkpoint.recv_string()
# there is something wrong with this
# in ami.client.flowchart.NodeProcess.send_checkpoint we send a
# fcMsgs.NodeCheckPoint but we are only ever receiving the state
new_node_state = await self.checkpoint.recv_pyobj()
node = self._graph.nodes[node_name]['node']
current_node_state = node.saveState()
restore_ctrl = False
restore_widget = False
if 'ctrl' in new_node_state:
if current_node_state['ctrl'] != new_node_state['ctrl']:
current_node_state['ctrl'] = new_node_state['ctrl']
restore_ctrl = True
if 'widget' in new_node_state:
if current_node_state['widget'] != new_node_state['widget']:
restore_widget = True
current_node_state['widget'] = new_node_state['widget']
if 'geometry' in new_node_state:
node.geometry = QtCore.QByteArray.fromHex(
bytes(new_node_state['geometry'], 'ascii'))
if restore_ctrl or restore_widget:
node.restoreState(current_node_state)
node.changed = node.isChanged(restore_ctrl, restore_widget)
if node.changed:
self.sigNodeChanged.emit(node)
node.viewed = new_node_state['viewed']
async def updateSources(self, init=False):
num_workers = None
while True:
topic = await self.graphinfo.recv_string()
source = await self.graphinfo.recv_string()
msg = await self.graphinfo.recv_pyobj()
if topic == 'sources':
source_library = SourceLibrary()
for source, node_type in msg.items():
pth = []
for part in source.split(':')[:-1]:
if pth:
part = ":".join((pth[-1], part))
pth.append(part)
source_library.addNodeType(source,
amitypes.loads(node_type),
[pth])
self.source_library = source_library
if init:
break
ctrl = self.widget()
tree = ctrl.ui.source_tree
ctrl.ui.clear_model(tree)
ctrl.ui.create_model(ctrl.ui.source_tree,
self.source_library.getLabelTree())
ctrl.chartWidget.updateStatus("Updated sources.")
elif topic == 'event_rate':
if num_workers is None:
ctrl = self.widget()
compiler_args = await ctrl.graphCommHandler.compilerArgs
num_workers = compiler_args['num_workers']
events_per_second = [None] * num_workers
total_events = [None] * num_workers
time_per_event = msg[ctrl.graph_name]
worker = int(re.search(r'(\d)+', source).group())
events_per_second[worker] = len(time_per_event) / (
time_per_event[-1][1] - time_per_event[0][0])
total_events[worker] = msg['num_events']
if all(events_per_second):
events_per_second = int(np.sum(events_per_second))
total_num_events = int(np.sum(total_events))
ctrl = self.widget()
ctrl.ui.rateLbl.setText(
f"Num Events: {total_num_events} Events/Sec: {events_per_second}"
)
events_per_second = [None] * num_workers
total_events = [None] * num_workers
elif topic == 'error':
ctrl = self.widget()
if hasattr(msg, 'node_name'):
node_name = ctrl.metadata[msg.node_name]['parent']
node = self.nodes(data='node')[node_name]
node.setException(msg)
ctrl.chartWidget.updateStatus(
f"{source} {node.name()}: {msg}", color='red')
else:
ctrl.chartWidget.updateStatus(f"{source}: {msg}",
color='red')
async def run(self):
await asyncio.gather(self.updateState(), self.updateSources())
class ExportWidget(QtWidgets.QWidget):
def __init__(self, node, text):
super().__init__()
self.node = node
self.text = text
self.setWindowTitle("Export")
self.layout = QtWidgets.QFormLayout(self)
self.setLayout(self.layout)
self.name = QtWidgets.QLineEdit(parent=self)
self.docstring = QtWidgets.QTextEdit(parent=self)
self.ok = QtWidgets.QPushButton("Ok", parent=self)
self.ok.clicked.connect(self.ok_clicked)
self.layout.addRow("Name:", self.name)
self.layout.addRow("Docstring:", self.docstring)
self.layout.addWidget(self.ok)
def ok_clicked(self):
self.fileDialog = FileDialog(None, "Save File..", '.', "Python (*.py)")
self.fileDialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
self.fileDialog.show()
self.fileDialog.fileSelected.connect(self.saveFile)
def saveFile(self, fileName):
node_name = self.name.text()
docstring = self.docstring.toPlainText()
terminals = {}
for name, term in self.node.terminals.items():
state = term.saveState()
state['ttype'] = fullname(term._type)
terminals[name] = state
template = self.export(node_name, docstring, terminals, self.text)
if not fileName.endswith('.py'):
fileName += '.py'
with open(fileName, 'w') as f:
f.write(template)
def export(self, name, docstring, terminals, text):
template = f"""
from typing import Any
from amitypes import Array1d, Array2d, Array3d
from ami.flowchart.Node import Node
import ami.graph_nodes as gn
{text}
class {name}(Node):
\"""
{docstring}
\"""
nodeName = "{name}"
def __init__(self, name):
super().__init__(name, terminals={terminals})
def to_operation(self, **kwargs):
proc = EventProcessor()
return gn.Map(name=self.name()+"_operation", **kwargs,
func=proc.on_event,
begin_run=proc.begin_run,
end_run=proc.end_run,
begin_step=proc.begin_step,
end_step=proc.end_step)
"""
return template
