def test_dynamic_link(self):
link = LinkItem()
anchor1 = AnchorPoint()
anchor2 = AnchorPoint()
self.scene.addItem(link)
self.scene.addItem(anchor1)
self.scene.addItem(anchor2)
link.setSourceItem(None, anchor1)
link.setSinkItem(None, anchor2)
anchor2.setPos(100, 100)
link.setSourceName("1")
link.setSinkName("2")
link.setDynamic(True)
self.assertTrue(link.isDynamic())
link.setDynamicEnabled(True)
self.assertTrue(link.isDynamicEnabled())
def advance():
clock = time.process_time()
link.setDynamic(clock > 1)
link.setDynamicEnabled(int(clock) % 2 == 0)
timer = QTimer(link, interval=0)
timer.timeout.connect(advance)
timer.start()
self.qWait()
timer.stop()
def test_arrowannotation(self):
item = ArrowItem()
self.scene.addItem(item)
item.setLine(QLineF(100, 100, 100, 200))
item.setLineWidth(5)
item = ArrowItem()
item.setLine(QLineF(150, 100, 150, 200))
item.setLineWidth(10)
item.setArrowStyle(ArrowItem.Concave)
self.scene.addItem(item)
item = ArrowAnnotation()
item.setPos(10, 10)
item.setLine(QLineF(10, 10, 200, 200))
self.scene.addItem(item)
item.setLineWidth(5)
def advance():
clock = time.process_time() * 10
item.setLineWidth(5 + math.sin(clock) * 5)
item.setColor(
QColor(Qt.red).lighter(100 + int(30 * math.cos(clock))))
timer = QTimer(item, interval=10)
timer.timeout.connect(advance)
timer.start()
self.qWait()
timer.stop()
class QCoreAppTestCase(unittest.TestCase):
_AppClass = QCoreApplication
@classmethod
def setUpClass(cls):
super(QCoreAppTestCase, cls).setUpClass()
app = cls._AppClass.instance()
if app is None:
app = cls._AppClass([])
cls.app = app
def setUp(self):
super(QCoreAppTestCase, self).setUp()
self._quittimer = QTimer(interval=1000)
self._quittimer.timeout.connect(self.app.quit)
self._quittimer.start()
def tearDown(self):
self._quittimer.stop()
self._quittimer.timeout.disconnect(self.app.quit)
self._quittimer = None
super(QCoreAppTestCase, self).tearDown()
@classmethod
def tearDownClass(cls):
gc.collect()
cls.app = None
super(QCoreAppTestCase, cls).tearDownClass()
class MagnetTool(DataTool):
"""
Draw points closer to the mouse position.
"""
def __init__(self, parent, plot):
super().__init__(parent, plot)
self.__timer = QTimer(self, interval=50)
self.__timer.timeout.connect(self.__timeout)
self._radius = 20.0
self._density = 4.0
self._pos = None
def mousePressEvent(self, event):
if event.button() == Qt.LeftButton:
self.editingStarted.emit()
self._pos = self.mapToPlot(event.pos())
self.__timer.start()
return True
return super().mousePressEvent(event)
def mouseMoveEvent(self, event):
if event.buttons() & Qt.LeftButton:
self._pos = self.mapToPlot(event.pos())
return True
return super().mouseMoveEvent(event)
def mouseReleaseEvent(self, event):
if event.button() == Qt.LeftButton:
self.__timer.stop()
self.editingFinished.emit()
return True
return super().mouseReleaseEvent(event)
def __timeout(self):
self.issueCommand.emit(Magnet(self._pos, self._radius, self._density))
def test_dock_standalone(self):
widget = QWidget()
layout = QHBoxLayout()
widget.setLayout(layout)
layout.addStretch(1)
widget.show()
dock = CollapsibleDockWidget()
layout.addWidget(dock)
list_view = QListView()
list_view.setModel(QStringListModel(["a", "b"], list_view))
label = QLabel("A label. ")
label.setWordWrap(True)
dock.setExpandedWidget(label)
dock.setCollapsedWidget(list_view)
dock.setExpanded(True)
dock.setExpanded(False)
timer = QTimer(dock, interval=50)
timer.timeout.connect(lambda: dock.setExpanded(not dock.expanded()))
timer.start()
self.qWait()
timer.stop()
def test_splashscreen(self):
splash = pkg_resources.resource_filename(
config.__package__, "icons/orange-canvas-core-splash.svg")
w = SplashScreen()
w.setPixmap(QPixmap(splash))
w.setTextRect(QRect(100, 100, 400, 50))
w.show()
def advance_time():
now = datetime.now()
time = now.strftime("%c : %f")
i = now.second % 3
if i == 2:
w.setTextFormat(Qt.RichText)
time = "<i>" + time + "</i>"
else:
w.setTextFormat(Qt.PlainText)
w.showMessage(time, alignment=Qt.AlignCenter)
rect = QRect(100, 100 + i * 20, 400, 50)
w.setTextRect(rect)
self.assertEqual(w.textRect(), rect)
timer = QTimer(w, interval=1)
timer.timeout.connect(advance_time)
timer.start()
self.qWait()
timer.stop()
def test_outputview(self):
output = OutputView()
output.show()
line1 = "A line \n"
line2 = "A different line\n"
output.write(line1)
self.assertEqual(output.toPlainText(), line1)
output.write(line2)
self.assertEqual(output.toPlainText(), line1 + line2)
output.clear()
self.assertEqual(output.toPlainText(), "")
output.writelines([line1, line2])
self.assertEqual(output.toPlainText(), line1 + line2)
output.setMaximumLines(5)
def advance():
now = datetime.now().strftime("%c\n")
output.write(now)
text = output.toPlainText()
self.assertLessEqual(len(text.splitlines()), 5)
timer = QTimer(output, interval=25)
timer.timeout.connect(advance)
timer.start()
self.qWait(100)
timer.stop()
class QSignalSpy(QObject):
"""
QSignalSpy(boundsignal)
"""
def __init__(self, boundsig, **kwargs):
super(QSignalSpy, self).__init__(**kwargs)
from AnyQt.QtCore import QEventLoop, QTimer
self.__boundsig = boundsig
self.__recorded = recorded = [] # type: List[List[Any]]
self.__loop = loop = QEventLoop()
self.__timer = QTimer(self, singleShot=True)
self.__timer.timeout.connect(self.__loop.quit)
def record(*args):
# Record the emitted arguments and quit the loop if running.
# NOTE: not capturing self from parent scope
recorded.append(list(args))
if loop.isRunning():
loop.quit()
# Need to keep reference at least for PyQt4 4.11.4, sip 4.16.9 on
# python 3.4 (if the signal is emitted during gc collection, and
# the boundsignal is a QObject.destroyed signal).
self.__record = record
boundsig.connect(record)
def signal(self):
return _QByteArray(self.__boundsig.signal[1:].encode("latin-1"))
def isValid(self):
return True
def wait(self, timeout=5000):
count = len(self)
self.__timer.stop()
self.__timer.setInterval(timeout)
self.__timer.start()
self.__loop.exec_()
self.__timer.stop()
return len(self) != count
def __getitem__(self, index):
return self.__recorded[index]
def __setitem__(self, index, value):
self.__recorded.__setitem__(index, value)
def __delitem__(self, index):
self.__recorded.__delitem__(index)
def __len__(self):
return len(self.__recorded)
def test_layout(self):
one_desc, negate_desc, cons_desc = self.widget_desc()
one_item = NodeItem()
one_item.setWidgetDescription(one_desc)
one_item.setPos(0, 150)
self.scene.add_node_item(one_item)
cons_item = NodeItem()
cons_item.setWidgetDescription(cons_desc)
cons_item.setPos(200, 0)
self.scene.add_node_item(cons_item)
negate_item = NodeItem()
negate_item.setWidgetDescription(negate_desc)
negate_item.setPos(200, 300)
self.scene.add_node_item(negate_item)
link = LinkItem()
link.setSourceItem(one_item)
link.setSinkItem(negate_item)
self.scene.add_link_item(link)
link = LinkItem()
link.setSourceItem(one_item)
link.setSinkItem(cons_item)
self.scene.add_link_item(link)
layout = AnchorLayout()
self.scene.addItem(layout)
self.scene.set_anchor_layout(layout)
layout.invalidateNode(one_item)
layout.activate()
p1, p2 = one_item.outputAnchorItem.anchorPositions()
self.assertTrue(p1 > p2)
self.scene.node_item_position_changed.connect(layout.invalidateNode)
path = QPainterPath()
path.addEllipse(125, 0, 50, 300)
def advance():
t = time.process_time()
cons_item.setPos(path.pointAtPercent(t % 1.0))
negate_item.setPos(path.pointAtPercent((t + 0.5) % 1.0))
timer = QTimer(negate_item, interval=5)
timer.start()
timer.timeout.connect(advance)
self.qWait()
timer.stop()
def test_framelesswindow(self):
window = FramelessWindow()
window.show()
window.setRadius(5)
def cycle():
window.setRadius((window.radius() + 3) % 30)
timer = QTimer(window, interval=50)
timer.timeout.connect(cycle)
timer.start()
self.qWait()
timer.stop()
def test_anchoritem(self):
anchoritem = NodeAnchorItem(None)
self.scene.addItem(anchoritem)
path = QPainterPath()
path.addEllipse(0, 0, 100, 100)
anchoritem.setAnchorPath(path)
anchor = AnchorPoint()
anchoritem.addAnchor(anchor)
ellipse1 = QGraphicsEllipseItem(-3, -3, 6, 6)
ellipse2 = QGraphicsEllipseItem(-3, -3, 6, 6)
self.scene.addItem(ellipse1)
self.scene.addItem(ellipse2)
anchor.scenePositionChanged.connect(ellipse1.setPos)
with self.assertRaises(ValueError):
anchoritem.addAnchor(anchor)
anchor1 = AnchorPoint()
anchoritem.addAnchor(anchor1)
anchor1.scenePositionChanged.connect(ellipse2.setPos)
self.assertSequenceEqual(anchoritem.anchorPoints(), [anchor, anchor1])
self.assertSequenceEqual(anchoritem.anchorPositions(), [0.5, 0.5])
anchoritem.setAnchorPositions([0.5, 0.0])
self.assertSequenceEqual(anchoritem.anchorPositions(), [0.5, 0.0])
def advance():
t = anchoritem.anchorPositions()
t = [(t + 0.05) % 1.0 for t in t]
anchoritem.setAnchorPositions(t)
timer = QTimer(anchoritem, interval=10)
timer.start()
timer.timeout.connect(advance)
self.qWait()
timer.stop()
class QSignalSpy(QObject):
"""
QSignalSpy(boundsignal)
"""
def __init__(self, boundsig, **kwargs):
super(QSignalSpy, self).__init__(**kwargs)
from AnyQt.QtCore import QEventLoop, QTimer
self.__boundsig = boundsig
self.__boundsig.connect(lambda *args: self.__record(*args))
self.__recorded = [] # type: List[List[Any]]
self.__loop = QEventLoop()
self.__timer = QTimer(self, singleShot=True)
self.__timer.timeout.connect(self.__loop.quit)
def __record(self, *args):
self.__recorded.append(list(args))
if self.__loop.isRunning():
self.__loop.quit()
def signal(self):
return _QByteArray(self.__boundsig.signal[1:].encode("latin-1"))
def isValid(self):
return True
def wait(self, timeout=5000):
count = len(self)
self.__timer.stop()
self.__timer.setInterval(timeout)
self.__timer.start()
self.__loop.exec_()
self.__timer.stop()
return len(self) != count
def __getitem__(self, index):
return self.__recorded[index]
def __setitem__(self, index, value):
self.__recorded.__setitem__(index, value)
def __delitem__(self, index):
self.__recorded.__delitem__(index)
def __len__(self):
return len(self.__recorded)
class PreviewModel(QStandardItemModel):
"""A model for preview items.
"""
def __init__(self, parent=None, items=None):
super().__init__(parent)
self.__preview_index = -1
if items is not None:
self.insertColumn(0, items)
self.__timer = QTimer(self)
self.__timer.timeout.connect(self.__process_next)
def delayedScanUpdate(self, delay=10):
"""Run a delayed preview item scan update.
"""
self.__preview_index = -1
self.__timer.start(delay)
log.debug("delayedScanUpdate: Start")
@Slot()
def __process_next(self):
index = self.__preview_index
log.debug("delayedScanUpdate: Next %i", index + 1)
if not 0 <= index + 1 < self.rowCount():
self.__timer.stop()
log.debug("delayedScanUpdate: Stop")
return
self.__preview_index = index = index + 1
assert 0 <= index < self.rowCount()
item = self.item(index)
if os.path.isfile(item.path()):
try:
scanner.scan_update(item)
except Exception:
log.error(
"An unexpected error occurred while "
"scanning '%s'.",
item.text(),
exc_info=True)
item.setEnabled(False)
class LightControl(QPushButton):
def __init__(self, *args, **kwargs):
self._strobe = kwargs.get('strobe', False)
self._strobeState = False
self._strobeTimer = QTimer()
self._strobeTimer.timeout.connect(self.__strobeEffect)
self._parent = kwargs.get('parent', None)
super(LightControl, self).__init__(*args)
self.clicked.connect(self.__callback)
self.setCheckable(True)
self.setIconSize(QSize(Config.iconSize, Config.iconSize))
self.setFixedSize(Config.controlWidth, Config.controlHeight)
self._lastPress = 0
def __callback(self):
self._strobeTimer.stop()
if self._parent is not None:
self._parent.setLight(self.text(), self.isChecked())
self._strobeState = False
if self._strobe:
now = nowMillis(datetime.datetime.now())
diff = now - self._lastPress
self._lastPress = now
if diff <= 500:
self.setChecked(True)
self._strobeTimer.start(Config.strobeRate / 1000)
def __strobeEffect(self):
if self._parent is not None:
self._strobeState = not self._strobeState
self._parent.setLight(self.text(), self._strobeState)
@property
def value(self):
return self._value
@value.setter
def value(self, value):
"""
Reject value updates.
"""
pass
class PreviewModel(QStandardItemModel):
"""A model for preview items.
"""
def __init__(self, parent=None, items=None):
QStandardItemModel.__init__(self, parent)
if items is not None:
self.insertColumn(0, items)
self.__timer = QTimer(self)
def delayedScanUpdate(self, delay=10):
"""Run a delayed preview item scan update.
"""
def iter_update(items):
for item in items:
try:
scanner.scan_update(item)
except Exception:
log.error(
"An unexpected error occurred while " "scanning %r.",
str(item.text()),
exc_info=True,
)
item.setEnabled(False)
yield
items = [self.item(i) for i in range(self.rowCount())]
iter_scan = iter_update(items)
def process_one():
try:
next(iter_scan)
except StopIteration:
self.__timer.timeout.disconnect(process_one)
self.__timer.stop()
self.__timer.timeout.connect(process_one)
self.__timer.start(delay)
class JitterTool(DataTool):
"""
Jitter points around the mouse position.
"""
def __init__(self, parent, plot):
super().__init__(parent, plot)
self.__timer = QTimer(self, interval=50)
self.__timer.timeout.connect(self._do)
self._pos = None
self._radius = 20.0
self._intensity = 5.0
self.__count = itertools.count()
def mousePressEvent(self, event):
if event.button() == Qt.LeftButton:
self.editingStarted.emit()
self._pos = self.mapToPlot(event.pos())
self.__timer.start()
return True
else:
return super().mousePressEvent(event)
def mouseMoveEvent(self, event):
if event.buttons() & Qt.LeftButton:
self._pos = self.mapToPlot(event.pos())
return True
else:
return super().mouseMoveEvent(event)
def mouseReleaseEvent(self, event):
if event.button() == Qt.LeftButton:
self.__timer.stop()
self.editingFinished.emit()
return True
else:
return super().mouseReleaseEvent(event)
def _do(self):
self.issueCommand.emit(
Jitter(self._pos, self._radius, self._intensity,
next(self.__count))
)
def test_dock_mainwinow(self):
mw = QMainWindow()
dock = CollapsibleDockWidget()
w1 = QTextEdit()
w2 = QToolButton()
w2.setFixedSize(38, 200)
dock.setExpandedWidget(w1)
dock.setCollapsedWidget(w2)
mw.addDockWidget(Qt.LeftDockWidgetArea, dock)
mw.setCentralWidget(QTextEdit())
mw.show()
timer = QTimer(dock, interval=50)
timer.timeout.connect(lambda: dock.setExpanded(not dock.expanded()))
timer.start()
self.qWait()
timer.stop()
class AirBrushTool(DataTool):
"""
Add points with an 'air brush'.
"""
only2d = False
def __init__(self, parent, plot):
super().__init__(parent, plot)
self.__timer = QTimer(self, interval=50)
self.__timer.timeout.connect(self.__timout)
self.__count = itertools.count()
self.__pos = None
def mousePressEvent(self, event):
if event.button() == Qt.LeftButton:
self.editingStarted.emit()
self.__pos = self.mapToPlot(event.pos())
self.__timer.start()
return True
else:
return super().mousePressEvent(event)
def mouseMoveEvent(self, event):
if event.buttons() & Qt.LeftButton:
self.__pos = self.mapToPlot(event.pos())
return True
else:
return super().mouseMoveEvent(event)
def mouseReleaseEvent(self, event):
if event.button() == Qt.LeftButton:
self.__timer.stop()
self.editingFinished.emit()
return True
else:
return super().mouseReleaseEvent(event)
def __timout(self):
self.issueCommand.emit(
AirBrush(self.__pos, None, None, next(self.__count))
)
class JitterTool(DataTool):
"""
Jitter points around the mouse position.
"""
def __init__(self, parent, plot):
super().__init__(parent, plot)
self.__timer = QTimer(self, interval=50)
self.__timer.timeout.connect(self._do)
self._pos = None
self._radius = 20.0
self._intensity = 5.0
self.__count = itertools.count()
def mousePressEvent(self, event):
if event.button() == Qt.LeftButton:
self.editingStarted.emit()
self._pos = self.mapToPlot(event.pos())
self.__timer.start()
return True
else:
return super().mousePressEvent(event)
def mouseMoveEvent(self, event):
if event.buttons() & Qt.LeftButton:
self._pos = self.mapToPlot(event.pos())
return True
else:
return super().mouseMoveEvent(event)
def mouseReleaseEvent(self, event):
if event.button() == Qt.LeftButton:
self.__timer.stop()
self.editingFinished.emit()
return True
else:
return super().mouseReleaseEvent(event)
def _do(self):
self.issueCommand.emit(
Jitter(self._pos, self._radius, self._intensity,
next(self.__count)))
def test_splitter_resizer(self):
w = QSplitter(orientation=Qt.Vertical)
w.addWidget(QWidget())
text = QTextEdit()
w.addWidget(text)
resizer = SplitterResizer(parent=None)
resizer.setSplitterAndWidget(w, text)
def toogle():
if resizer.size() == 0:
resizer.open()
else:
resizer.close()
w.show()
timer = QTimer(resizer, interval=100)
timer.timeout.connect(toogle)
timer.start()
toogle()
self.qWait()
timer.stop()
class AirBrushTool(DataTool):
"""
Add points with an 'air brush'.
"""
only2d = False
def __init__(self, parent, plot):
super().__init__(parent, plot)
self.__timer = QTimer(self, interval=50)
self.__timer.timeout.connect(self.__timout)
self.__count = itertools.count()
self.__pos = None
def mousePressEvent(self, event):
if event.button() == Qt.LeftButton:
self.editingStarted.emit()
self.__pos = self.mapToPlot(event.pos())
self.__timer.start()
return True
else:
return super().mousePressEvent(event)
def mouseMoveEvent(self, event):
if event.buttons() & Qt.LeftButton:
self.__pos = self.mapToPlot(event.pos())
return True
else:
return super().mouseMoveEvent(event)
def mouseReleaseEvent(self, event):
if event.button() == Qt.LeftButton:
self.__timer.stop()
self.editingFinished.emit()
return True
else:
return super().mouseReleaseEvent(event)
def __timout(self):
self.issueCommand.emit(
AirBrush(self.__pos, None, None, next(self.__count)))
class MagnetTool(DataTool):
"""
Draw points closer to the mouse position.
"""
def __init__(self, parent, plot):
super().__init__(parent, plot)
self.__timer = QTimer(self, interval=50)
self.__timer.timeout.connect(self.__timeout)
self._radius = 20.0
self._density = 4.0
self._pos = None
def mousePressEvent(self, event):
if event.button() == Qt.LeftButton:
self.editingStarted.emit()
self._pos = self.mapToPlot(event.pos())
self.__timer.start()
return True
else:
return super().mousePressEvent(event)
def mouseMoveEvent(self, event):
if event.buttons() & Qt.LeftButton:
self._pos = self.mapToPlot(event.pos())
return True
else:
return super().mouseMoveEvent(event)
def mouseReleaseEvent(self, event):
if event.button() == Qt.LeftButton:
self.__timer.stop()
self.editingFinished.emit()
return True
else:
return super().mouseReleaseEvent(event)
def __timeout(self):
self.issueCommand.emit(
Magnet(self._pos, self._radius, self._density)
)
class QCoreAppTestCase(unittest.TestCase):
_AppClass = QCoreApplication
app = None # type: QCoreApplication
__appdomain = ""
__appname = ""
@classmethod
def setUpClass(cls):
super(QCoreAppTestCase, cls).setUpClass()
QStandardPaths.setTestModeEnabled(True)
app = cls._AppClass.instance()
if app is None:
app = cls._AppClass([])
cls.app = app
cls.__appname = cls.app.applicationName()
cls.__appdomain = cls.app.organizationDomain()
cls.app.setApplicationName("orangecanvas.testing")
cls.app.setOrganizationDomain("biolab.si")
def setUp(self):
super(QCoreAppTestCase, self).setUp()
self._quittimer = QTimer(interval=100)
self._quittimer.timeout.connect(self.app.quit)
self._quittimer.start()
def tearDown(self):
self._quittimer.stop()
self._quittimer.timeout.disconnect(self.app.quit)
self._quittimer = None
super(QCoreAppTestCase, self).tearDown()
@classmethod
def tearDownClass(cls):
gc.collect()
cls.app.setApplicationName(cls.__appname)
cls.app.setOrganizationDomain(cls.__appdomain)
cls.app = None
super(QCoreAppTestCase, cls).tearDownClass()
QStandardPaths.setTestModeEnabled(False)
class PreviewModel(QStandardItemModel):
"""A model for preview items.
"""
def __init__(self, parent=None, items=None):
QStandardItemModel.__init__(self, parent)
if items is not None:
self.insertColumn(0, items)
self.__timer = QTimer(self)
def delayedScanUpdate(self, delay=10):
"""Run a delayed preview item scan update.
"""
def iter_update(items):
for item in items:
try:
scanner.scan_update(item)
except Exception:
log.error("An unexpected error occurred while "
"scanning %r.", str(item.text()),
exc_info=True)
item.setEnabled(False)
yield
items = [self.item(i) for i in range(self.rowCount())]
iter_scan = iter_update(items)
def process_one():
try:
next(iter_scan)
except StopIteration:
self.__timer.timeout.disconnect(process_one)
self.__timer.stop()
self.__timer.timeout.connect(process_one)
self.__timer.start(delay)
class SignalManager(QObject):
"""
Handle all runtime signal propagation for a :clas:`Scheme` instance.
The scheme must be passed to the constructor and will become the parent
of this object. Furthermore this should happen before any items
(nodes, links) are added to the scheme.
"""
Running, Stoped, Paused, Error = range(4)
"""SignalManger state flags."""
Waiting, Processing = range(2)
"""SignalManager runtime state flags."""
stateChanged = Signal(int)
"""Emitted when the state of the signal manager changes."""
updatesPending = Signal()
"""Emitted when signals are added to the queue."""
processingStarted = Signal([], [SchemeNode])
"""Emitted right before a `SchemeNode` instance has its inputs
updated.
"""
processingFinished = Signal([], [SchemeNode])
"""Emitted right after a `SchemeNode` instance has had its inputs
updated.
"""
runtimeStateChanged = Signal(int)
"""Emitted when `SignalManager`'s runtime state changes."""
def __init__(self, scheme):
assert scheme
QObject.__init__(self, scheme)
self._input_queue = []
# mapping a node to it's current outputs
# {node: {channel: {id: signal_value}}}
self._node_outputs = {}
self.__state = SignalManager.Running
self.__runtime_state = SignalManager.Waiting
# A flag indicating if UpdateRequest event should be rescheduled
self.__reschedule = False
self.__update_timer = QTimer(self, interval=100, singleShot=True)
self.__update_timer.timeout.connect(self.__process_next)
def _can_process(self):
"""
Return a bool indicating if the manger can enter the main
processing loop.
"""
return self.__state not in [SignalManager.Error, SignalManager.Stoped]
def scheme(self):
"""
Return the parent class:`Scheme` instance.
"""
return self.parent()
def start(self):
"""
Start the update loop.
.. note:: The updates will not happen until the control reaches
the Qt event loop.
"""
if self.__state != SignalManager.Running:
self.__state = SignalManager.Running
self.stateChanged.emit(SignalManager.Running)
self._update()
def stop(self):
"""
Stop the update loop.
.. note:: If the `SignalManager` is currently in `process_queues` it
will still update all current pending signals, but will not
re-enter until `start()` is called again
"""
if self.__state != SignalManager.Stoped:
self.__state = SignalManager.Stoped
self.stateChanged.emit(SignalManager.Stoped)
self.__update_timer.stop()
def pause(self):
"""
Pause the updates.
"""
if self.__state != SignalManager.Paused:
self.__state = SignalManager.Paused
self.stateChanged.emit(SignalManager.Paused)
self.__update_timer.stop()
def resume(self):
if self.__state == SignalManager.Paused:
self.__state = SignalManager.Running
self.stateChanged.emit(self.__state)
self._update()
def step(self):
if self.__state == SignalManager.Paused:
self.process_queued()
def state(self):
"""
Return the current state.
"""
return self.__state
def _set_runtime_state(self, state):
"""
Set the runtime state.
Should only be called by `SignalManager` implementations.
"""
if self.__runtime_state != state:
self.__runtime_state = state
self.runtimeStateChanged.emit(self.__runtime_state)
def runtime_state(self):
"""
Return the runtime state. This can be `SignalManager.Waiting`
or `SignalManager.Processing`.
"""
return self.__runtime_state
def on_node_removed(self, node):
# remove all pending input signals for node so we don't get
# stale references in process_node.
# NOTE: This does not remove output signals for this node. In
# particular the final 'None' will be delivered to the sink
# nodes even after the source node is no longer in the scheme.
log.info("Node %r removed. Removing pending signals.", node.title)
self.remove_pending_signals(node)
del self._node_outputs[node]
def on_node_added(self, node):
self._node_outputs[node] = defaultdict(dict)
def link_added(self, link):
# push all current source values to the sink
link.set_runtime_state(SchemeLink.Empty)
if link.enabled:
log.info("Link added (%s). Scheduling signal data update.", link)
self._schedule(self.signals_on_link(link))
self._update()
link.enabled_changed.connect(self.link_enabled_changed)
def link_removed(self, link):
# purge all values in sink's queue
log.info("Link removed (%s). Scheduling signal data purge.", link)
self.purge_link(link)
link.enabled_changed.disconnect(self.link_enabled_changed)
def link_enabled_changed(self, enabled):
if enabled:
link = self.sender()
log.info("Link %s enabled. Scheduling signal data update.", link)
self._schedule(self.signals_on_link(link))
def signals_on_link(self, link):
"""
Return _Signal instances representing the current values
present on the link.
"""
items = self.link_contents(link)
signals = []
for key, value in list(items.items()):
signals.append(_Signal(link, value, key))
return signals
def link_contents(self, link):
"""
Return the contents on link.
"""
node, channel = link.source_node, link.source_channel
if node in self._node_outputs:
return self._node_outputs[node][channel]
else:
# if the the node was already removed it's tracked outputs in
# _node_outputs are cleared, however the final 'None' signal
# deliveries for the link are left in the _input_queue.
pending = [sig for sig in self._input_queue if sig.link is link]
return {sig.id: sig.value for sig in pending}
def send(self, node, channel, value, id):
"""
"""
log.debug("%r sending %r (id: %r) on channel %r", node.title, type(value), id, channel.name)
scheme = self.scheme()
self._node_outputs[node][channel][id] = value
links = scheme.find_links(source_node=node, source_channel=channel)
links = list(filter(is_enabled, links))
signals = []
for link in links:
signals.append(_Signal(link, value, id))
self._schedule(signals)
def purge_link(self, link):
"""
Purge the link (send None for all ids currently present)
"""
contents = self.link_contents(link)
ids = list(contents.keys())
signals = [_Signal(link, None, id) for id in ids]
self._schedule(signals)
def _schedule(self, signals):
"""
Schedule a list of :class:`_Signal` for delivery.
"""
self._input_queue.extend(signals)
for link in {sig.link for sig in signals}:
# update the SchemeLink's runtime state flags
contents = self.link_contents(link)
if any(value is not None for value in contents.values()):
state = SchemeLink.Active
else:
state = SchemeLink.Empty
link.set_runtime_state(state | SchemeLink.Pending)
if signals:
self.updatesPending.emit()
self._update()
def _update_link(self, link):
"""
Schedule update of a single link.
"""
signals = self.signals_on_link(link)
self._schedule(signals)
def process_queued(self, max_nodes=None):
"""
Process queued signals.
Take one node node from the pending input queue and deliver
all scheduled signals.
"""
if max_nodes is not None or max_nodes != 1:
warnings.warn(
"`max_nodes` is deprecated and unused (will always equal 1)", DeprecationWarning, stacklevel=2
)
if self.__runtime_state == SignalManager.Processing:
raise RuntimeError("Cannot re-enter 'process_queued'")
if not self._can_process():
raise RuntimeError("Can't process in state %i" % self.__state)
log.info("SignalManager: Processing queued signals")
node_update_front = self.node_update_front()
log.debug("SignalManager: Nodes eligible for update %s", [node.title for node in node_update_front])
if node_update_front:
node = node_update_front[0]
self._set_runtime_state(SignalManager.Processing)
try:
self.process_node(node)
finally:
self._set_runtime_state(SignalManager.Waiting)
def process_node(self, node):
"""
Process pending input signals for `node`.
"""
signals_in = self.pending_input_signals(node)
self.remove_pending_signals(node)
signals_in = self.compress_signals(signals_in)
log.debug("Processing %r, sending %i signals.", node.title, len(signals_in))
# Clear the link's pending flag.
for link in {sig.link for sig in signals_in}:
link.set_runtime_state(link.runtime_state() & ~SchemeLink.Pending)
assert {sig.link for sig in self._input_queue}.intersection({sig.link for sig in signals_in}) == set([])
self.processingStarted.emit()
self.processingStarted[SchemeNode].emit(node)
try:
self.send_to_node(node, signals_in)
finally:
self.processingFinished.emit()
self.processingFinished[SchemeNode].emit(node)
def compress_signals(self, signals):
"""
Compress a list of :class:`_Signal` instances to be delivered.
The base implementation returns the list unmodified.
"""
return signals
def send_to_node(self, node, signals):
"""
Abstract. Reimplement in subclass.
Send/notify the :class:`SchemeNode` instance (or whatever
object/instance it is a representation of) that it has new inputs
as represented by the signals list (list of :class:`_Signal`).
"""
raise NotImplementedError
def is_pending(self, node):
"""
Is `node` (class:`SchemeNode`) scheduled for processing (i.e.
it has incoming pending signals).
"""
return node in [signal.link.sink_node for signal in self._input_queue]
def pending_nodes(self):
"""
Return a list of pending nodes.
The nodes are returned in the order they were enqueued for
signal delivery.
Returns
-------
nodes : List[SchemeNode]
"""
return list(unique(sig.link.sink_node for sig in self._input_queue))
def pending_input_signals(self, node):
"""
Return a list of pending input signals for node.
"""
return [signal for signal in self._input_queue if node is signal.link.sink_node]
def remove_pending_signals(self, node):
"""
Remove pending signals for `node`.
"""
for signal in self.pending_input_signals(node):
try:
self._input_queue.remove(signal)
except ValueError:
pass
def blocking_nodes(self):
"""
Return a list of nodes in a blocking state.
"""
scheme = self.scheme()
return [node for node in scheme.nodes if self.is_blocking(node)]
def is_blocking(self, node):
return False
def node_update_front(self):
"""
Return a list of nodes on the update front, i.e. nodes scheduled for
an update that have no ancestor which is either itself scheduled
for update or is in a blocking state)
.. note::
The node's ancestors are only computed over enabled links.
"""
scheme = self.scheme()
blocking_nodes = set(self.blocking_nodes())
dependents = partial(dependent_nodes, scheme)
blocked_nodes = reduce(set.union, map(dependents, blocking_nodes), set(blocking_nodes))
pending = self.pending_nodes()
pending_downstream = reduce(set.union, map(dependents, pending), set())
log.debug("Pending nodes: %s", pending)
log.debug("Blocking nodes: %s", blocking_nodes)
noneligible = pending_downstream | blocked_nodes
return [node for node in pending if node not in noneligible]
@Slot()
def __process_next(self):
if not self.__state == SignalManager.Running:
log.debug("Received 'UpdateRequest' while not in 'Running' state")
return
if self.__runtime_state == SignalManager.Processing:
# This happens if someone calls QCoreApplication.processEvents
# from the signal handlers.
# A `__process_next` must be rescheduled when exiting
# process_queued.
log.warning(
"Received 'UpdateRequest' while in 'process_queued'. "
"An update will be re-scheduled when exiting the "
"current update."
)
self.__reschedule = True
return
log.info("'UpdateRequest' event, queued signals: %i", len(self._input_queue))
if self._input_queue:
self.process_queued()
if self.__reschedule and self.__state == SignalManager.Running:
self.__reschedule = False
log.debug("Rescheduling signal update")
self.__update_timer.start()
if self.node_update_front():
log.debug("More nodes are eligible for an update. " "Scheduling another update.")
self._update()
def _update(self):
"""
Schedule processing at a later time.
"""
if self.__state == SignalManager.Running and not self.__update_timer.isActive():
self.__update_timer.start()
class OWGOEnrichmentAnalysis(widget.OWWidget):
name = "GO Browser"
description = "Enrichment analysis for Gene Ontology terms."
icon = "../widgets/icons/GOBrowser.svg"
priority = 2020
inputs = [("Cluster Data", Orange.data.Table,
"setDataset", widget.Single + widget.Default),
("Reference Data", Orange.data.Table,
"setReferenceDataset")]
outputs = [("Data on Selected Genes", Orange.data.Table),
("Data on Unselected Genes", Orange.data.Table),
("Data on Unknown Genes", Orange.data.Table),
("Enrichment Report", Orange.data.Table)]
settingsHandler = settings.DomainContextHandler()
annotationIndex = settings.ContextSetting(0)
geneAttrIndex = settings.ContextSetting(0)
useAttrNames = settings.ContextSetting(False)
geneMatcherSettings = settings.Setting([True, False, False, False])
useReferenceDataset = settings.Setting(False)
aspectIndex = settings.Setting(0)
useEvidenceType = settings.Setting(
{et: True for et in go.evidenceTypesOrdered})
filterByNumOfInstances = settings.Setting(False)
minNumOfInstances = settings.Setting(1)
filterByPValue = settings.Setting(True)
maxPValue = settings.Setting(0.2)
filterByPValue_nofdr = settings.Setting(False)
maxPValue_nofdr = settings.Setting(0.01)
probFunc = settings.Setting(0)
selectionDirectAnnotation = settings.Setting(0)
selectionDisjoint = settings.Setting(0)
selectionAddTermAsClass = settings.Setting(0)
def __init__(self, parent=None):
super().__init__(self, parent)
self.clusterDataset = None
self.referenceDataset = None
self.ontology = None
self.annotations = None
self.loadedAnnotationCode = None
self.treeStructRootKey = None
self.probFunctions = [stats.Binomial(), stats.Hypergeometric()]
self.selectedTerms = []
self.selectionChanging = 0
self.__state = State.Ready
self.__scheduletimer = QTimer(self, singleShot=True)
self.__scheduletimer.timeout.connect(self.__update)
#############
## GUI
#############
self.tabs = gui.tabWidget(self.controlArea)
## Input tab
self.inputTab = gui.createTabPage(self.tabs, "Input")
box = gui.widgetBox(self.inputTab, "Info")
self.infoLabel = gui.widgetLabel(box, "No data on input\n")
gui.button(box, self, "Ontology/Annotation Info",
callback=self.ShowInfo,
tooltip="Show information on loaded ontology and annotations")
box = gui.widgetBox(self.inputTab, "Organism")
self.annotationComboBox = gui.comboBox(
box, self, "annotationIndex", items=[],
callback=self.__invalidateAnnotations, tooltip="Select organism"
)
genebox = gui.widgetBox(self.inputTab, "Gene Names")
self.geneAttrIndexCombo = gui.comboBox(
genebox, self, "geneAttrIndex", callback=self.__invalidate,
tooltip="Use this attribute to extract gene names from input data")
self.geneAttrIndexCombo.setDisabled(self.useAttrNames)
cb = gui.checkBox(genebox, self, "useAttrNames", "Use column names",
tooltip="Use column names for gene names",
callback=self.__invalidate)
cb.toggled[bool].connect(self.geneAttrIndexCombo.setDisabled)
gui.button(genebox, self, "Gene matcher settings",
callback=self.UpdateGeneMatcher,
tooltip="Open gene matching settings dialog")
self.referenceRadioBox = gui.radioButtonsInBox(
self.inputTab, self, "useReferenceDataset",
["Entire genome", "Reference set (input)"],
tooltips=["Use entire genome for reference",
"Use genes from Referece Examples input signal as reference"],
box="Reference", callback=self.__invalidate)
self.referenceRadioBox.buttons[1].setDisabled(True)
gui.radioButtonsInBox(
self.inputTab, self, "aspectIndex",
["Biological process", "Cellular component", "Molecular function"],
box="Aspect", callback=self.__invalidate)
## Filter tab
self.filterTab = gui.createTabPage(self.tabs, "Filter")
box = gui.widgetBox(self.filterTab, "Filter GO Term Nodes")
gui.checkBox(box, self, "filterByNumOfInstances", "Genes",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by number of input genes mapped to a term")
ibox = gui.indentedBox(box)
gui.spin(ibox, self, 'minNumOfInstances', 1, 100,
step=1, label='#:', labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Min. number of input genes mapped to a term")
gui.checkBox(box, self, "filterByPValue_nofdr", "p-value",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by term p-value")
gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue_nofdr', 1e-8, 1,
step=1e-8, label='p:', labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Max term p-value")
#use filterByPValue for FDR, as it was the default in prior versions
gui.checkBox(box, self, "filterByPValue", "FDR",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by term FDR")
gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue', 1e-8, 1,
step=1e-8, label='p:', labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Max term p-value")
box = gui.widgetBox(box, "Significance test")
gui.radioButtonsInBox(box, self, "probFunc", ["Binomial", "Hypergeometric"],
tooltips=["Use binomial distribution test",
"Use hypergeometric distribution test"],
callback=self.__invalidate) # TODO: only update the p values
box = gui.widgetBox(self.filterTab, "Evidence codes in annotation",
addSpace=True)
self.evidenceCheckBoxDict = {}
for etype in go.evidenceTypesOrdered:
ecb = QCheckBox(
etype, toolTip=go.evidenceTypes[etype],
checked=self.useEvidenceType[etype])
ecb.toggled.connect(self.__on_evidenceChanged)
box.layout().addWidget(ecb)
self.evidenceCheckBoxDict[etype] = ecb
## Select tab
self.selectTab = gui.createTabPage(self.tabs, "Select")
box = gui.radioButtonsInBox(
self.selectTab, self, "selectionDirectAnnotation",
["Directly or Indirectly", "Directly"],
box="Annotated genes",
callback=self.ExampleSelection)
box = gui.widgetBox(self.selectTab, "Output", addSpace=True)
gui.radioButtonsInBox(
box, self, "selectionDisjoint",
btnLabels=["All selected genes",
"Term-specific genes",
"Common term genes"],
tooltips=["Outputs genes annotated to all selected GO terms",
"Outputs genes that appear in only one of selected GO terms",
"Outputs genes common to all selected GO terms"],
callback=[self.ExampleSelection,
self.UpdateAddClassButton])
self.addClassCB = gui.checkBox(
box, self, "selectionAddTermAsClass", "Add GO Term as class",
callback=self.ExampleSelection)
# ListView for DAG, and table for significant GOIDs
self.DAGcolumns = ['GO term', 'Cluster', 'Reference', 'p-value',
'FDR', 'Genes', 'Enrichment']
self.splitter = QSplitter(Qt.Vertical, self.mainArea)
self.mainArea.layout().addWidget(self.splitter)
# list view
self.listView = GOTreeWidget(self.splitter)
self.listView.setSelectionMode(QTreeView.ExtendedSelection)
self.listView.setAllColumnsShowFocus(1)
self.listView.setColumnCount(len(self.DAGcolumns))
self.listView.setHeaderLabels(self.DAGcolumns)
self.listView.header().setSectionsClickable(True)
self.listView.header().setSortIndicatorShown(True)
self.listView.setSortingEnabled(True)
self.listView.setItemDelegateForColumn(
6, EnrichmentColumnItemDelegate(self))
self.listView.setRootIsDecorated(True)
self.listView.itemSelectionChanged.connect(self.ViewSelectionChanged)
# table of significant GO terms
self.sigTerms = QTreeWidget(self.splitter)
self.sigTerms.setColumnCount(len(self.DAGcolumns))
self.sigTerms.setHeaderLabels(self.DAGcolumns)
self.sigTerms.setSortingEnabled(True)
self.sigTerms.setSelectionMode(QTreeView.ExtendedSelection)
self.sigTerms.setItemDelegateForColumn(
6, EnrichmentColumnItemDelegate(self))
self.sigTerms.itemSelectionChanged.connect(self.TableSelectionChanged)
self.sigTableTermsSorted = []
self.graph = {}
self.originalGraph = None
self.inputTab.layout().addStretch(1)
self.filterTab.layout().addStretch(1)
self.selectTab.layout().addStretch(1)
class AnnotationSlot(SimpleNamespace):
taxid = ... # type: str
name = ... # type: str
orgcode = ... # type: str
filename = ... # type:str
available_annotations = [
AnnotationSlot(
taxid=taxid,
name=taxonomy.common_taxid_to_name(taxid),
orgcode=go.from_taxid(taxid),
filename="gene_association.{}.tar.gz"
.format(go.from_taxid(taxid))
)
for taxid in taxonomy.common_taxids()
if go.from_taxid(taxid)
]
self.availableAnnotations = sorted(
available_annotations, key=lambda a: a.name
)
self.annotationComboBox.clear()
for a in self.availableAnnotations:
self.annotationComboBox.addItem(a.name)
self.annotationComboBox.setCurrentIndex(self.annotationIndex)
self.annotationIndex = self.annotationComboBox.currentIndex()
self._executor = ThreadExecutor()
def sizeHint(self):
return QSize(1000, 700)
def __on_evidenceChanged(self):
for etype, cb in self.evidenceCheckBoxDict.items():
self.useEvidenceType[etype] = cb.isChecked()
self.__invalidate()
def UpdateGeneMatcher(self):
"""Open the Gene matcher settings dialog."""
dialog = GeneMatcherDialog(self, defaults=self.geneMatcherSettings, modal=True)
if dialog.exec_() != QDialog.Rejected:
self.geneMatcherSettings = [getattr(dialog, item[0]) for item in dialog.items]
self.__invalidateAnnotations()
def clear(self):
self.infoLabel.setText("No data on input\n")
self.warning(0)
self.warning(1)
self.geneAttrIndexCombo.clear()
self.ClearGraph()
self.send("Data on Selected Genes", None)
self.send("Data on Unselected Genes", None)
self.send("Data on Unknown Genes", None)
self.send("Enrichment Report", None)
def setDataset(self, data=None):
self.closeContext()
self.clear()
self.clusterDataset = data
if data is not None:
domain = data.domain
allvars = domain.variables + domain.metas
self.candidateGeneAttrs = [var for var in allvars if isstring(var)]
self.geneAttrIndexCombo.clear()
for var in self.candidateGeneAttrs:
self.geneAttrIndexCombo.addItem(*gui.attributeItem(var))
taxid = data_hints.get_hint(data, "taxid", "")
try:
code = go.from_taxid(taxid)
except KeyError:
pass
else:
_c2i = {a.orgcode: i
for i, a in enumerate(self.availableAnnotations)}
try:
self.annotationIndex = _c2i[code]
except KeyError:
pass
self.useAttrNames = data_hints.get_hint(data, "genesinrows",
self.useAttrNames)
self.openContext(data)
self.geneAttrIndex = min(self.geneAttrIndex,
len(self.candidateGeneAttrs) - 1)
if len(self.candidateGeneAttrs) == 0:
self.useAttrNames = True
self.geneAttrIndex = -1
elif self.geneAttrIndex < len(self.candidateGeneAttrs):
self.geneAttrIndex = len(self.candidateGeneAttrs) - 1
self.__invalidate()
def setReferenceDataset(self, data=None):
self.referenceDataset = data
self.referenceRadioBox.buttons[1].setDisabled(not bool(data))
self.referenceRadioBox.buttons[1].setText("Reference set")
if self.clusterDataset is not None and self.useReferenceDataset:
self.useReferenceDataset = 0 if not data else 1
self.__invalidate()
elif self.clusterDataset:
self.__updateReferenceSetButton()
def handleNewSignals(self):
super().handleNewSignals()
self.__update()
@Slot()
def __invalidate(self):
# Invalidate the current results or pending task and schedule an
# update.
self.__scheduletimer.start()
if self.__state != State.Ready:
self.__state |= State.Stale
self.SetGraph({})
self.referenceGenes = None
self.clusterGenes = None
def __invalidateAnnotations(self):
self.annotations = None
self.loadedAnnotationCode = None
if self.clusterDataset:
self.infoLabel.setText("...\n")
self.__updateReferenceSetButton()
self.__invalidate()
@Slot()
def __update(self):
self.__scheduletimer.stop()
if self.clusterDataset is None:
return
if self.__state & State.Running:
self.__state |= State.Stale
elif self.__state & State.Downloading:
self.__state |= State.Stale
elif self.__state & State.Ready:
if self.__ensure_data():
self.Load()
self.Enrichment()
else:
assert self.__state & State.Downloading
assert self.isBlocking()
def __updateReferenceSetButton(self):
allgenes, refgenes = None, None
if self.referenceDataset and self.annotations is not None:
try:
allgenes = self.genesFromTable(self.referenceDataset)
except Exception:
allgenes = []
refgenes, unknown = self.FilterAnnotatedGenes(allgenes)
self.referenceRadioBox.buttons[1].setDisabled(not bool(allgenes))
self.referenceRadioBox.buttons[1].setText("Reference set " + ("(%i genes, %i matched)" % (len(allgenes), len(refgenes)) if allgenes and refgenes else ""))
def genesFromTable(self, data):
if self.useAttrNames:
genes = [v.name for v in data.domain.variables]
else:
attr = self.candidateGeneAttrs[min(self.geneAttrIndex, len(self.candidateGeneAttrs) - 1)]
genes = [str(ex[attr]) for ex in data if not numpy.isnan(ex[attr])]
if any("," in gene for gene in genes):
self.information(0, "Separators detected in gene names. Assuming multiple genes per example.")
genes = reduce(operator.iadd, (genes.split(",") for genes in genes), [])
return genes
def FilterAnnotatedGenes(self, genes):
matchedgenes = self.annotations.get_gene_names_translator(genes).values()
return matchedgenes, [gene for gene in genes if gene not in matchedgenes]
def FilterUnknownGenes(self):
if not self.useAttrNames and self.candidateGeneAttrs:
geneAttr = self.candidateGeneAttrs[min(self.geneAttrIndex, len(self.candidateGeneAttrs)-1)]
indices = []
for i, ex in enumerate(self.clusterDataset):
if not any(self.annotations.genematcher.match(n.strip()) for n in str(ex[geneAttr]).split(",")):
indices.append(i)
if indices:
data = self.clusterDataset[indices]
else:
data = None
self.send("Data on Unknown Genes", data)
else:
self.send("Data on Unknown Genes", None)
def __start_download(self, files_list):
# type: (List[Tuple[str, str]]) -> None
task = EnsureDownloaded(files_list)
task.progress.connect(self._progressBarSet)
f = self._executor.submit(task)
fw = FutureWatcher(f, self)
fw.finished.connect(self.__download_finish)
fw.finished.connect(fw.deleteLater)
fw.resultReady.connect(self.__invalidate)
self.progressBarInit(processEvents=None)
self.setBlocking(True)
self.setStatusMessage("Downloading")
self.__state = State.Downloading
@Slot(Future)
def __download_finish(self, result):
# type: (Future[None]) -> None
assert QThread.currentThread() is self.thread()
assert result.done()
self.setBlocking(False)
self.setStatusMessage("")
self.progressBarFinished(processEvents=False)
try:
result.result()
except ConnectTimeout:
logging.getLogger(__name__).error("Error:")
self.error(2, "Internet connection error, unable to load data. " +
"Check connection and create a new GO Browser widget.")
except RequestException as err:
logging.getLogger(__name__).error("Error:")
self.error(2, "Internet error:\n" + str(err))
except BaseException as err:
logging.getLogger(__name__).error("Error:")
self.error(2, "Error:\n" + str(err))
raise
else:
self.error(2)
finally:
self.__state = State.Ready
def __ensure_data(self):
# Ensure that all required database (ontology and annotations for
# the current selected organism are present. If not start a download in
# the background. Return True if all dbs are present and false
# otherwise
assert self.__state == State.Ready
annotation = self.availableAnnotations[self.annotationIndex]
go_files = serverfiles.listfiles("GO")
files = []
if annotation.filename not in go_files:
files.append(("GO", annotation.filename))
if go.Ontology.FILENAME not in go_files:
files.append((go.Ontology.DOMAIN, go.Ontology.FILENAME))
gene_info_files = serverfiles.listfiles(gene.NCBIGeneInfo.DOMAIN)
gi_filename = gene.NCBIGeneInfo.FILENAME.format(
taxid=gene.NCBIGeneInfo.TAX_MAP.get(
annotation.taxid, annotation.taxid)
)
if gi_filename not in gene_info_files and self.geneMatcherSettings[2]:
files.append((gene.NCBIGeneInfo.DOMAIN, gi_filename))
if files:
self.__start_download(files)
assert self.__state == State.Downloading
return False
else:
return True
def Load(self):
a = self.availableAnnotations[self.annotationIndex]
if self.ontology is None:
self.ontology = go.Ontology()
if a.orgcode != self.loadedAnnotationCode:
self.annotations = None
gc.collect() # Force run garbage collection
self.annotations = go.Annotations(a.orgcode, genematcher=gene.GMDirect())
self.loadedAnnotationCode = a.orgcode
count = defaultdict(int)
geneSets = defaultdict(set)
for anno in self.annotations.annotations:
count[anno.evidence] += 1
geneSets[anno.evidence].add(anno.geneName)
for etype in go.evidenceTypesOrdered:
ecb = self.evidenceCheckBoxDict[etype]
ecb.setEnabled(bool(count[etype]))
ecb.setText(etype + ": %i annots(%i genes)" % (count[etype], len(geneSets[etype])))
self.__updateReferenceSetButton()
def SetGeneMatcher(self):
if self.annotations:
taxid = self.annotations.taxid
matchers = []
for matcher, use in zip([gene.GMGO, gene.GMKEGG, gene.GMNCBI, gene.GMAffy], self.geneMatcherSettings):
if use:
try:
if taxid == "352472":
matchers.extend([matcher(taxid), gene.GMDicty(),
[matcher(taxid), gene.GMDicty()]])
# The reason machers are duplicated is that we want `matcher` or `GMDicty` to
# match genes by them self if possible. Only use the joint matcher if they fail.
else:
matchers.append(matcher(taxid))
except Exception as ex:
print(ex)
self.annotations.genematcher = gene.matcher(matchers)
self.annotations.genematcher.set_targets(self.annotations.gene_names)
self.__updateReferenceSetButton()
def Enrichment(self):
assert self.clusterDataset is not None
assert self.__state == State.Ready
if not self.annotations.ontology:
self.annotations.ontology = self.ontology
if isinstance(self.annotations.genematcher, gene.GMDirect):
self.SetGeneMatcher()
self.error(1)
self.warning([0, 1])
if self.useAttrNames:
clusterGenes = [v.name for v in self.clusterDataset.domain.attributes]
self.information(0)
elif 0 <= self.geneAttrIndex < len(self.candidateGeneAttrs):
geneAttr = self.candidateGeneAttrs[self.geneAttrIndex]
clusterGenes = [str(ex[geneAttr]) for ex in self.clusterDataset
if not numpy.isnan(ex[geneAttr])]
if any("," in gene for gene in clusterGenes):
self.information(0, "Separators detected in cluster gene names. Assuming multiple genes per example.")
clusterGenes = reduce(operator.iadd, (genes.split(",") for genes in clusterGenes), [])
else:
self.information(0)
else:
self.error(1, "Failed to extract gene names from input dataset!")
return {}
genesSetCount = len(set(clusterGenes))
self.clusterGenes = clusterGenes = self.annotations.get_gene_names_translator(clusterGenes).values()
self.infoLabel.setText("%i unique genes on input\n%i (%.1f%%) genes with known annotations" % (genesSetCount, len(clusterGenes), 100.0*len(clusterGenes)/genesSetCount if genesSetCount else 0.0))
referenceGenes = None
if not self.useReferenceDataset or self.referenceDataset is None:
self.information(2)
self.information(1)
referenceGenes = self.annotations.gene_names
elif self.referenceDataset is not None:
if self.useAttrNames:
referenceGenes = [v.name for v in self.referenceDataset.domain.attributes]
self.information(1)
elif geneAttr in (self.referenceDataset.domain.variables +
self.referenceDataset.domain.metas):
referenceGenes = [str(ex[geneAttr]) for ex in self.referenceDataset
if not numpy.isnan(ex[geneAttr])]
if any("," in gene for gene in clusterGenes):
self.information(1, "Separators detected in reference gene names. Assuming multiple genes per example.")
referenceGenes = reduce(operator.iadd, (genes.split(",") for genes in referenceGenes), [])
else:
self.information(1)
else:
self.information(1)
referenceGenes = None
if referenceGenes is None:
referenceGenes = list(self.annotations.gene_names)
self.referenceRadioBox.buttons[1].setText("Reference set")
self.referenceRadioBox.buttons[1].setDisabled(True)
self.information(2, "Unable to extract gene names from reference dataset. Using entire genome for reference")
self.useReferenceDataset = 0
else:
refc = len(referenceGenes)
referenceGenes = self.annotations.get_gene_names_translator(referenceGenes).values()
self.referenceRadioBox.buttons[1].setText("Reference set (%i genes, %i matched)" % (refc, len(referenceGenes)))
self.referenceRadioBox.buttons[1].setDisabled(False)
self.information(2)
else:
self.useReferenceDataset = 0
if not referenceGenes:
self.error(1, "No valid reference set")
return {}
self.referenceGenes = referenceGenes
evidences = []
for etype in go.evidenceTypesOrdered:
if self.useEvidenceType[etype]:
evidences.append(etype)
aspect = ["P", "C", "F"][self.aspectIndex]
self.progressBarInit(processEvents=False)
self.setBlocking(True)
self.__state = State.Running
if clusterGenes:
f = self._executor.submit(
self.annotations.get_enriched_terms,
clusterGenes, referenceGenes, evidences, aspect=aspect,
prob=self.probFunctions[self.probFunc], use_fdr=False,
progress_callback=methodinvoke(
self, "_progressBarSet", (float,))
)
fw = FutureWatcher(f, parent=self)
fw.done.connect(self.__on_enrichment_done)
fw.done.connect(fw.deleteLater)
return
else:
f = Future()
f.set_result({})
self.__on_enrichment_done(f)
def __on_enrichment_done(self, results):
# type: (Future[Dict[str, tuple]]) -> None
self.progressBarFinished(processEvents=False)
self.setBlocking(False)
self.setStatusMessage("")
if self.__state & State.Stale:
self.__state = State.Ready
self.__invalidate()
return
self.__state = State.Ready
try:
results = results.result() # type: Dict[str, tuple]
except Exception as ex:
results = {}
error = str(ex)
self.error(1, error)
if results:
terms = list(results.items())
fdr_vals = stats.FDR([d[1] for _, d in terms])
terms = [(key, d + (fdr,))
for (key, d), fdr in zip(terms, fdr_vals)]
terms = dict(terms)
else:
terms = {}
self.terms = terms
if not self.terms:
self.warning(0, "No enriched terms found.")
else:
self.warning(0)
self.treeStructDict = {}
ids = self.terms.keys()
self.treeStructRootKey = None
parents = {}
for id in ids:
parents[id] = set([term for _, term in self.ontology[id].related])
children = {}
for term in self.terms:
children[term] = set([id for id in ids if term in parents[id]])
for term in self.terms:
self.treeStructDict[term] = TreeNode(self.terms[term], children[term])
if not self.ontology[term].related and not getattr(self.ontology[term], "is_obsolete", False):
self.treeStructRootKey = term
self.FilterUnknownGenes()
self.SetGraph(terms)
self._updateEnrichmentReportOutput()
self.commit()
def _updateEnrichmentReportOutput(self):
terms = sorted(self.terms.items(), key=lambda item: item[1][1])
# Create and send the enrichemnt report table.
termsDomain = Orange.data.Domain(
[], [],
# All is meta!
[Orange.data.StringVariable("GO Term Id"),
Orange.data.StringVariable("GO Term Name"),
Orange.data.ContinuousVariable("Cluster Frequency"),
Orange.data.ContinuousVariable("Genes in Cluster",
number_of_decimals=0),
Orange.data.ContinuousVariable("Reference Frequency"),
Orange.data.ContinuousVariable("Genes in Reference",
number_of_decimals=0),
Orange.data.ContinuousVariable("p-value"),
Orange.data.ContinuousVariable("FDR"),
Orange.data.ContinuousVariable("Enrichment"),
Orange.data.StringVariable("Genes")])
terms = [[t_id,
self.ontology[t_id].name,
len(genes) / len(self.clusterGenes),
len(genes),
r_count / len(self.referenceGenes),
r_count,
p_value,
fdr,
len(genes) / len(self.clusterGenes) * \
len(self.referenceGenes) / r_count,
",".join(genes)
]
for t_id, (genes, p_value, r_count, fdr) in terms
if genes and r_count]
if terms:
X = numpy.empty((len(terms), 0))
M = numpy.array(terms, dtype=object)
termsTable = Orange.data.Table.from_numpy(termsDomain, X,
metas=M)
else:
termsTable = None
self.send("Enrichment Report", termsTable)
@Slot(float)
def _progressBarSet(self, value):
assert QThread.currentThread() is self.thread()
self.progressBarSet(value, processEvents=None)
@Slot()
def _progressBarFinish(self):
assert QThread.currentThread() is self.thread()
self.progressBarFinished(processEvents=None)
def FilterGraph(self, graph):
if self.filterByPValue_nofdr:
graph = go.filterByPValue(graph, self.maxPValue_nofdr)
if self.filterByPValue: #FDR
graph = dict(filter(lambda item: item[1][3] <= self.maxPValue, graph.items()))
if self.filterByNumOfInstances:
graph = dict(filter(lambda item: len(item[1][0]) >= self.minNumOfInstances, graph.items()))
return graph
def FilterAndDisplayGraph(self):
if self.clusterDataset and self.originalGraph is not None:
self.graph = self.FilterGraph(self.originalGraph)
if self.originalGraph and not self.graph:
self.warning(1, "All found terms were filtered out.")
else:
self.warning(1)
self.ClearGraph()
self.DisplayGraph()
def SetGraph(self, graph=None):
self.originalGraph = graph
if graph:
self.FilterAndDisplayGraph()
else:
self.graph = {}
self.ClearGraph()
def ClearGraph(self):
self.listView.clear()
self.listViewItems=[]
self.sigTerms.clear()
def DisplayGraph(self):
fromParentDict = {}
self.termListViewItemDict = {}
self.listViewItems = []
enrichment = lambda t: len(t[0]) / t[2] * (len(self.referenceGenes) / len(self.clusterGenes))
maxFoldEnrichment = max([enrichment(term) for term in self.graph.values()] or [1])
def addNode(term, parent, parentDisplayNode):
if (parent, term) in fromParentDict:
return
if term in self.graph:
displayNode = GOTreeWidgetItem(self.ontology[term], self.graph[term], len(self.clusterGenes), len(self.referenceGenes), maxFoldEnrichment, parentDisplayNode)
displayNode.goId = term
self.listViewItems.append(displayNode)
if term in self.termListViewItemDict:
self.termListViewItemDict[term].append(displayNode)
else:
self.termListViewItemDict[term] = [displayNode]
fromParentDict[(parent, term)] = True
parent = term
else:
displayNode = parentDisplayNode
for c in self.treeStructDict[term].children:
addNode(c, parent, displayNode)
if self.treeStructDict:
addNode(self.treeStructRootKey, None, self.listView)
terms = self.graph.items()
terms = sorted(terms, key=lambda item: item[1][1])
self.sigTableTermsSorted = [t[0] for t in terms]
self.sigTerms.clear()
for i, (t_id, (genes, p_value, refCount, fdr)) in enumerate(terms):
item = GOTreeWidgetItem(self.ontology[t_id],
(genes, p_value, refCount, fdr),
len(self.clusterGenes),
len(self.referenceGenes),
maxFoldEnrichment,
self.sigTerms)
item.goId = t_id
self.listView.expandAll()
for i in range(5):
self.listView.resizeColumnToContents(i)
self.sigTerms.resizeColumnToContents(i)
self.sigTerms.resizeColumnToContents(6)
width = min(self.listView.columnWidth(0), 350)
self.listView.setColumnWidth(0, width)
self.sigTerms.setColumnWidth(0, width)
def ViewSelectionChanged(self):
if self.selectionChanging:
return
self.selectionChanging = 1
self.selectedTerms = []
selected = self.listView.selectedItems()
self.selectedTerms = list(set([lvi.term.id for lvi in selected]))
self.ExampleSelection()
self.selectionChanging = 0
def TableSelectionChanged(self):
if self.selectionChanging:
return
self.selectionChanging = 1
self.selectedTerms = []
selectedIds = set([self.sigTerms.itemFromIndex(index).goId for index in self.sigTerms.selectedIndexes()])
for i in range(self.sigTerms.topLevelItemCount()):
item = self.sigTerms.topLevelItem(i)
selected = item.goId in selectedIds
term = item.goId
if selected:
self.selectedTerms.append(term)
for lvi in self.termListViewItemDict[term]:
try:
lvi.setSelected(selected)
if selected:
lvi.setExpanded(True)
except RuntimeError: # Underlying C/C++ object deleted
pass
self.selectionChanging = 0
self.ExampleSelection()
def UpdateAddClassButton(self):
self.addClassCB.setEnabled(self.selectionDisjoint == 1)
def ExampleSelection(self):
self.commit()
def commit(self):
if self.clusterDataset is None or self.originalGraph is None or \
self.annotations is None:
return
if self.__state & State.Stale:
return
terms = set(self.selectedTerms)
genes = reduce(operator.ior,
(set(self.graph[term][0]) for term in terms), set())
evidences = []
for etype in go.evidenceTypesOrdered:
if self.useEvidenceType[etype]:
evidences.append(etype)
allTerms = self.annotations.get_annotated_terms(
genes, direct_annotation_only=self.selectionDirectAnnotation,
evidence_codes=evidences)
if self.selectionDisjoint > 0:
count = defaultdict(int)
for term in self.selectedTerms:
for g in allTerms.get(term, []):
count[g] += 1
ccount = 1 if self.selectionDisjoint == 1 else len(self.selectedTerms)
selectedGenes = [gene for gene, c in count.items()
if c == ccount and gene in genes]
else:
selectedGenes = reduce(
operator.ior,
(set(allTerms.get(term, [])) for term in self.selectedTerms),
set())
if self.useAttrNames:
vars = [self.clusterDataset.domain[gene]
for gene in set(selectedGenes)]
domain = Orange.data.Domain(
vars, self.clusterDataset.domain.class_vars,
self.clusterDataset.domain.metas)
newdata = self.clusterDataset.from_table(domain, self.clusterDataset)
self.send("Data on Selected Genes", newdata)
self.send("Data on Unselected Genes", None)
elif self.candidateGeneAttrs:
selectedExamples = []
unselectedExamples = []
geneAttr = self.candidateGeneAttrs[min(self.geneAttrIndex, len(self.candidateGeneAttrs)-1)]
if self.selectionDisjoint == 1:
goVar = Orange.data.DiscreteVariable(
"GO Term", values=list(self.selectedTerms))
newDomain = Orange.data.Domain(
self.clusterDataset.domain.variables, goVar,
self.clusterDataset.domain.metas)
goColumn = []
for i, ex in enumerate(self.clusterDataset):
if not numpy.isnan(ex[geneAttr]) and any(gene in selectedGenes for gene in str(ex[geneAttr]).split(",")):
if self.selectionDisjoint == 1 and self.selectionAddTermAsClass:
terms = filter(lambda term: any(gene in self.graph[term][0] for gene in str(ex[geneAttr]).split(",")) , self.selectedTerms)
term = sorted(terms)[0]
goColumn.append(goVar.values.index(term))
selectedExamples.append(i)
else:
unselectedExamples.append(i)
if selectedExamples:
selectedExamples = self.clusterDataset[selectedExamples]
if self.selectionDisjoint == 1 and self.selectionAddTermAsClass:
selectedExamples = Orange.data.Table.from_table(newDomain, selectedExamples)
view, issparse = selectedExamples.get_column_view(goVar)
assert not issparse
view[:] = goColumn
else:
selectedExamples = None
if unselectedExamples:
unselectedExamples = self.clusterDataset[unselectedExamples]
else:
unselectedExamples = None
self.send("Data on Selected Genes", selectedExamples)
self.send("Data on Unselected Genes", unselectedExamples)
def ShowInfo(self):
dialog = QDialog(self)
dialog.setModal(False)
dialog.setLayout(QVBoxLayout())
label = QLabel(dialog)
label.setText("Ontology:\n" + self.ontology.header
if self.ontology else "Ontology not loaded!")
dialog.layout().addWidget(label)
label = QLabel(dialog)
label.setText("Annotations:\n" + self.annotations.header.replace("!", "")
if self.annotations else "Annotations not loaded!")
dialog.layout().addWidget(label)
dialog.show()
def onDeleteWidget(self):
"""Called before the widget is removed from the canvas.
"""
self.annotations = None
self.ontology = None
gc.collect() # Force collection
class WidgetManager(QObject):
"""
OWWidget instance manager class.
This class handles the lifetime of OWWidget instances in a
:class:`WidgetsScheme`.
"""
#: A new OWWidget was created and added by the manager.
widget_for_node_added = Signal(SchemeNode, QWidget)
#: An OWWidget was removed, hidden and will be deleted when appropriate.
widget_for_node_removed = Signal(SchemeNode, QWidget)
class ProcessingState(enum.IntEnum):
"""Widget processing state flags"""
#: Signal manager is updating/setting the widget's inputs
InputUpdate = 1
#: Widget has entered a blocking state (OWWidget.isBlocking)
BlockingUpdate = 2
#: Widget has entered processing state
ProcessingUpdate = 4
#: Widget is still in the process of initialization
Initializing = 8
InputUpdate, BlockingUpdate, ProcessingUpdate, Initializing = ProcessingState
#: Widget initialization states
Delayed = namedtuple(
"Delayed", ["node"])
PartiallyInitialized = namedtuple(
"Materializing",
["node", "partially_initialized_widget"])
Materialized = namedtuple(
"Materialized",
["node", "widget"])
class CreationPolicy(enum.Enum):
"""Widget Creation Policy"""
#: Widgets are scheduled to be created from the event loop, or when
#: first accessed with `widget_for_node`
Normal = "Normal"
#: Widgets are created immediately when added to the workflow model
Immediate = "Immediate"
#: Widgets are created only when first accessed with `widget_for_node`
OnDemand = "OnDemand"
Normal, Immediate, OnDemand = CreationPolicy
def __init__(self, parent=None):
QObject.__init__(self, parent)
self.__scheme = None
self.__signal_manager = None
self.__widgets = []
self.__initstate_for_node = {}
self.__creation_policy = WidgetManager.Normal
#: a queue of all nodes whose widgets are scheduled for
#: creation/initialization
self.__init_queue = deque() # type: Deque[SchemeNode]
#: Timer for scheduling widget initialization
self.__init_timer = QTimer(self, interval=0, singleShot=True)
self.__init_timer.timeout.connect(self.__create_delayed)
#: A mapping of SchemeNode -> OWWidget (note: a mapping is only added
#: after the widget is actually created)
self.__widget_for_node = {}
#: a mapping of OWWidget -> SchemeNode
self.__node_for_widget = {}
# Widgets that were 'removed' from the scheme but were at
# the time in an input update loop and could not be deleted
# immediately
self.__delay_delete = set()
#: Deleted/removed during creation/initialization.
self.__delete_after_create = []
#: processing state flags for all widgets (including the ones
#: in __delay_delete).
#: Note: widgets which have not yet been created do not have an entry
self.__widget_processing_state = {}
# Tracks the widget in the update loop by the SignalManager
self.__updating_widget = None
def set_scheme(self, scheme):
"""
Set the :class:`WidgetsScheme` instance to manage.
"""
self.__scheme = scheme
self.__signal_manager = scheme.findChild(SignalManager)
self.__signal_manager.processingStarted[SchemeNode].connect(
self.__on_processing_started
)
self.__signal_manager.processingFinished[SchemeNode].connect(
self.__on_processing_finished
)
scheme.node_added.connect(self.add_widget_for_node)
scheme.node_removed.connect(self.remove_widget_for_node)
scheme.runtime_env_changed.connect(self.__on_env_changed)
scheme.installEventFilter(self)
def scheme(self):
"""
Return the scheme instance on which this manager is installed.
"""
return self.__scheme
def signal_manager(self):
"""
Return the signal manager in use on the :func:`scheme`.
"""
return self.__signal_manager
def widget_for_node(self, node):
"""
Return the OWWidget instance for the scheme node.
"""
state = self.__initstate_for_node[node]
if isinstance(state, WidgetManager.Delayed):
# Create the widget now if it is still pending
state = self.__materialize(state)
return state.widget
elif isinstance(state, WidgetManager.PartiallyInitialized):
widget = state.partially_initialized_widget
log.warning("WidgetManager.widget_for_node: "
"Accessing a partially created widget instance. "
"This is most likely a result of explicit "
"QApplication.processEvents call from the '%s.%s' "
"widgets __init__.",
type(widget).__module__, type(widget).__name__)
return widget
elif isinstance(state, WidgetManager.Materialized):
return state.widget
else:
assert False
def node_for_widget(self, widget):
"""
Return the SchemeNode instance for the OWWidget.
Raise a KeyError if the widget does not map to a node in the scheme.
"""
return self.__node_for_widget[widget]
def widget_properties(self, node):
"""
Return the current widget properties/settings.
Parameters
----------
node : SchemeNode
Returns
-------
settings : dict
"""
state = self.__initstate_for_node[node]
if isinstance(state, WidgetManager.Materialized):
return state.widget.settingsHandler.pack_data(state.widget)
else:
return node.properties
def set_creation_policy(self, policy):
"""
Set the widget creation policy
Parameters
----------
policy : WidgetManager.CreationPolicy
"""
if self.__creation_policy != policy:
self.__creation_policy = policy
if self.__creation_policy == WidgetManager.Immediate:
self.__init_timer.stop()
while self.__init_queue:
state = self.__init_queue.popleft()
self.__materialize(state)
elif self.__creation_policy == WidgetManager.Normal:
if not self.__init_timer.isActive() and self.__init_queue:
self.__init_timer.start()
elif self.__creation_policy == WidgetManager.OnDemand:
self.__init_timer.stop()
else:
assert False
def creation_policy(self):
"""
Return the current widget creation policy
Returns
-------
policy: WidgetManager.CreationPolicy
"""
return self.__creation_policy
def add_widget_for_node(self, node):
"""
Create a new OWWidget instance for the corresponding scheme node.
"""
state = WidgetManager.Delayed(node)
self.__initstate_for_node[node] = state
if self.__creation_policy == WidgetManager.Immediate:
self.__initstate_for_node[node] = self.__materialize(state)
elif self.__creation_policy == WidgetManager.Normal:
self.__init_queue.append(state)
if not self.__init_timer.isActive():
self.__init_timer.start()
elif self.__creation_policy == WidgetManager.OnDemand:
self.__init_queue.append(state)
def __materialize(self, state):
# Create and initialize an OWWidget for a Delayed
# widget initialization
assert isinstance(state, WidgetManager.Delayed)
if state in self.__init_queue:
self.__init_queue.remove(state)
node = state.node
widget = self.create_widget_instance(node)
self.__widgets.append(widget)
self.__widget_for_node[node] = widget
self.__node_for_widget[widget] = node
self.__initialize_widget_state(node, widget)
state = WidgetManager.Materialized(node, widget)
self.__initstate_for_node[node] = state
self.widget_for_node_added.emit(node, widget)
return state
def remove_widget_for_node(self, node):
"""
Remove the OWWidget instance for node.
"""
state = self.__initstate_for_node[node]
if isinstance(state, WidgetManager.Delayed):
del self.__initstate_for_node[node]
self.__init_queue.remove(state)
elif isinstance(state, WidgetManager.Materialized):
# Update the node's stored settings/properties dict before
# removing the widget.
# TODO: Update/sync whenever the widget settings change.
node.properties = self._widget_settings(state.widget)
self.__widgets.remove(state.widget)
del self.__initstate_for_node[node]
del self.__widget_for_node[node]
del self.__node_for_widget[state.widget]
node.title_changed.disconnect(state.widget.setCaption)
state.widget.progressBarValueChanged.disconnect(node.set_progress)
self.widget_for_node_removed.emit(node, state.widget)
self._delete_widget(state.widget)
elif isinstance(state, WidgetManager.PartiallyInitialized):
widget = state.partially_initialized_widget
raise RuntimeError(
"A widget/node {} was removed while being initialized. "
"This is most likely a result of an explicit "
"QApplication.processEvents call from the '{}.{}' "
"widgets __init__.\n"
.format(state.node.title, type(widget).__module__,
type(widget).__init__))
def _widget_settings(self, widget):
return widget.settingsHandler.pack_data(widget)
def _delete_widget(self, widget):
"""
Delete the OWBaseWidget instance.
"""
widget.close()
# Save settings to user global settings.
widget.saveSettings()
# Notify the widget it will be deleted.
widget.onDeleteWidget()
if self.__widget_processing_state[widget] != 0:
# If the widget is in an update loop and/or blocking we
# delay the scheduled deletion until the widget is done.
self.__delay_delete.add(widget)
else:
widget.deleteLater()
del self.__widget_processing_state[widget]
def create_widget_instance(self, node):
"""
Create a OWWidget instance for the node.
"""
desc = node.description
klass = widget = None
initialized = False
error = None
# First try to actually retrieve the class.
try:
klass = name_lookup(desc.qualified_name)
except (ImportError, AttributeError):
sys.excepthook(*sys.exc_info())
error = "Could not import {0!r}\n\n{1}".format(
node.description.qualified_name, traceback.format_exc()
)
except Exception:
sys.excepthook(*sys.exc_info())
error = "An unexpected error during import of {0!r}\n\n{1}".format(
node.description.qualified_name, traceback.format_exc()
)
if klass is None:
widget = mock_error_owwidget(node, error)
initialized = True
if widget is None:
log.info("WidgetManager: Creating '%s.%s' instance '%s'.",
klass.__module__, klass.__name__, node.title)
widget = klass.__new__(
klass,
None,
captionTitle=node.title,
signal_manager=self.signal_manager(),
stored_settings=node.properties,
# NOTE: env is a view of the real env and reflects
# changes to the environment.
env=self.scheme().runtime_env()
)
initialized = False
# Init the node/widget mapping and state before calling __init__
# Some OWWidgets might already send data in the constructor
# (should this be forbidden? Raise a warning?) triggering the signal
# manager which would request the widget => node mapping or state
# Furthermore they can (though they REALLY REALLY REALLY should not)
# explicitly call qApp.processEvents.
assert node not in self.__widget_for_node
self.__widget_for_node[node] = widget
self.__node_for_widget[widget] = node
self.__widget_processing_state[widget] = WidgetManager.Initializing
self.__initstate_for_node[node] = \
WidgetManager.PartiallyInitialized(node, widget)
if not initialized:
try:
widget.__init__()
except Exception:
sys.excepthook(*sys.exc_info())
msg = traceback.format_exc()
msg = "Could not create {0!r}\n\n{1}".format(
node.description.name, msg
)
# remove state tracking for widget ...
del self.__widget_for_node[node]
del self.__node_for_widget[widget]
del self.__widget_processing_state[widget]
# ... and substitute it with a mock error widget.
widget = mock_error_owwidget(node, msg)
self.__widget_for_node[node] = widget
self.__node_for_widget[widget] = node
self.__widget_processing_state[widget] = 0
self.__initstate_for_node[node] = \
WidgetManager.Materialized(node, widget)
self.__initstate_for_node[node] = \
WidgetManager.Materialized(node, widget)
# Clear Initializing flag
self.__widget_processing_state[widget] &= ~WidgetManager.Initializing
node.title_changed.connect(widget.setCaption)
# Widget's info/warning/error messages.
widget.messageActivated.connect(self.__on_widget_state_changed)
widget.messageDeactivated.connect(self.__on_widget_state_changed)
# Widget's statusTip
node.set_status_message(widget.statusMessage())
widget.statusMessageChanged.connect(node.set_status_message)
# Widget's progress bar value state.
widget.progressBarValueChanged.connect(node.set_progress)
# Widget processing state (progressBarInit/Finished)
# and the blocking state.
widget.processingStateChanged.connect(
self.__on_processing_state_changed
)
widget.blockingStateChanged.connect(self.__on_blocking_state_changed)
if widget.isBlocking():
# A widget can already enter blocking state in __init__
self.__widget_processing_state[widget] |= self.BlockingUpdate
if widget.processingState != 0:
# It can also start processing (initialization of resources, ...)
self.__widget_processing_state[widget] |= self.ProcessingUpdate
node.set_processing_state(1)
node.set_progress(widget.progressBarValue)
# Install a help shortcut on the widget
help_shortcut = QShortcut(QKeySequence("F1"), widget)
help_shortcut.activated.connect(self.__on_help_request)
# Up shortcut (activate/open parent)
up_shortcut = QShortcut(
QKeySequence(Qt.ControlModifier + Qt.Key_Up), widget)
up_shortcut.activated.connect(self.__on_activate_parent)
# Call setters only after initialization.
widget.setWindowIcon(
icon_loader.from_description(desc).get(desc.icon)
)
widget.setCaption(node.title)
# Schedule an update with the signal manager, due to the cleared
# implicit Initializing flag
self.signal_manager()._update()
return widget
def node_processing_state(self, node):
"""
Return the processing state flags for the node.
Same as `manager.widget_processing_state(manger.widget_for_node(node))`
"""
state = self.__initstate_for_node[node]
if isinstance(state, WidgetManager.Materialized):
return self.__widget_processing_state[state.widget]
elif isinstance(state, WidgetManager.PartiallyInitialized):
return self.__widget_processing_state[state.partially_initialized_widget]
else:
return WidgetManager.Initializing
def widget_processing_state(self, widget):
"""
Return the processing state flags for the widget.
The state is an bitwise or of `InputUpdate` and `BlockingUpdate`.
"""
return self.__widget_processing_state[widget]
def __create_delayed(self):
if self.__init_queue:
state = self.__init_queue.popleft()
node = state.node
self.__initstate_for_node[node] = self.__materialize(state)
if self.__creation_policy == WidgetManager.Normal and \
self.__init_queue:
# restart the timer if pending widgets still in the queue
self.__init_timer.start()
def eventFilter(self, receiver, event):
if event.type() == QEvent.Close and receiver is self.__scheme:
self.signal_manager().stop()
# Notify the widget instances.
for widget in list(self.__widget_for_node.values()):
widget.close()
widget.saveSettings()
widget.onDeleteWidget()
event.accept()
return True
return QObject.eventFilter(self, receiver, event)
def __on_help_request(self):
"""
Help shortcut was pressed. We send a `QWhatsThisClickedEvent` to
the scheme and hope someone responds to it.
"""
# Sender is the QShortcut, and parent the OWBaseWidget
widget = self.sender().parent()
try:
node = self.node_for_widget(widget)
except KeyError:
pass
else:
qualified_name = node.description.qualified_name
help_url = "help://search?" + urlencode({"id": qualified_name})
event = QWhatsThisClickedEvent(help_url)
QCoreApplication.sendEvent(self.scheme(), event)
def __on_activate_parent(self):
"""
Activate parent shortcut was pressed.
"""
event = ActivateParentEvent()
QCoreApplication.sendEvent(self.scheme(), event)
def __initialize_widget_state(self, node, widget):
"""
Initialize the tracked info/warning/error message state.
"""
for message_group in widget.message_groups:
message = user_message_from_state(message_group)
if message:
node.set_state_message(message)
def __on_widget_state_changed(self, msg):
"""
The OWBaseWidget info/warning/error state has changed.
"""
widget = msg.group.widget
try:
node = self.node_for_widget(widget)
except KeyError:
pass
else:
self.__initialize_widget_state(node, widget)
def __on_processing_state_changed(self, state):
"""
A widget processing state has changed (progressBarInit/Finished)
"""
widget = self.sender()
try:
node = self.node_for_widget(widget)
except KeyError:
return
if state:
self.__widget_processing_state[widget] |= self.ProcessingUpdate
else:
self.__widget_processing_state[widget] &= ~self.ProcessingUpdate
self.__update_node_processing_state(node)
def __on_processing_started(self, node):
"""
Signal manager entered the input update loop for the node.
"""
widget = self.widget_for_node(node)
# Remember the widget instance. The node and the node->widget mapping
# can be removed between this and __on_processing_finished.
self.__updating_widget = widget
self.__widget_processing_state[widget] |= self.InputUpdate
self.__update_node_processing_state(node)
def __on_processing_finished(self, node):
"""
Signal manager exited the input update loop for the node.
"""
widget = self.__updating_widget
self.__widget_processing_state[widget] &= ~self.InputUpdate
if widget in self.__node_for_widget:
self.__update_node_processing_state(node)
elif widget in self.__delay_delete:
self.__try_delete(widget)
else:
raise ValueError("%r is not managed" % widget)
self.__updating_widget = None
def __on_blocking_state_changed(self, state):
"""
OWWidget blocking state has changed.
"""
if not state:
# schedule an update pass.
self.signal_manager()._update()
widget = self.sender()
if state:
self.__widget_processing_state[widget] |= self.BlockingUpdate
else:
self.__widget_processing_state[widget] &= ~self.BlockingUpdate
if widget in self.__node_for_widget:
node = self.node_for_widget(widget)
self.__update_node_processing_state(node)
elif widget in self.__delay_delete:
self.__try_delete(widget)
def __update_node_processing_state(self, node):
"""
Update the `node.processing_state` to reflect the widget state.
"""
state = self.node_processing_state(node)
node.set_processing_state(1 if state else 0)
def __try_delete(self, widget):
if self.__widget_processing_state[widget] == 0:
self.__delay_delete.remove(widget)
widget.deleteLater()
del self.__widget_processing_state[widget]
def __on_env_changed(self, key, newvalue, oldvalue):
# Notify widgets of a runtime environment change
for widget in self.__widget_for_node.values():
widget.workflowEnvChanged(key, newvalue, oldvalue)
class QuickHelp(QTextBrowser):
#: Emitted when the shown text changes.
textChanged = Signal()
def __init__(self, *args, **kwargs):
QTextBrowser.__init__(self, *args, **kwargs)
self.setOpenExternalLinks(False)
self.setOpenLinks(False)
self.__text = ""
self.__permanentText = ""
self.__defaultText = ""
self.__timer = QTimer(self, timeout=self.__on_timeout,
singleShot=True)
self.anchorClicked.connect(self.__on_anchorClicked)
def showHelp(self, text, timeout=0):
"""
Show help for `timeout` milliseconds. if timeout is 0 then
show the text until it is cleared with clearHelp or showHelp is
called with an empty string.
"""
if self.__text != text:
self.__text = str(text)
self.__update()
self.textChanged.emit()
if timeout > 0:
self.__timer.start(timeout)
def clearHelp(self):
"""
Clear help text previously set with `showHelp`.
"""
self.__timer.stop()
self.showHelp("")
def showPermanentHelp(self, text):
"""
Set permanent help text. The text may be temporarily overridden
by showHelp but will be shown again when that is cleared.
"""
if self.__permanentText != text:
self.__permanentText = text
self.__update()
self.textChanged.emit()
def setDefaultText(self, text):
"""
Set default help text. The text is overriden by normal and permanent help messages,
but is show again after such messages are cleared.
"""
if self.__defaultText != text:
self.__defaultText = text
self.__update()
self.textChanged.emit()
def currentText(self):
"""
Return the current shown text.
"""
return self.__text or self.__permanentText
def __update(self):
if self.__text:
self.setHtml(self.__text)
elif self.__permanentText:
self.setHtml(self.__permanentText)
else:
self.setHtml(self.__defaultText)
def __on_timeout(self):
if self.__text:
self.__text = ""
self.__update()
self.textChanged.emit()
def __on_anchorClicked(self, anchor):
ev = QuickHelpDetailRequestEvent(anchor.toString(), anchor)
QCoreApplication.postEvent(self, ev)
class WidgetManager(QObject):
"""
WidgetManager class is responsible for creation, tracking and deletion
of UI elements constituting an interactive workflow.
It does so by reacting to changes in the underlying workflow model,
creating and destroying the components when needed.
This is an abstract class, subclassed MUST reimplement at least
:func:`create_widget_for_node` and :func:`delete_widget_for_node`.
The widgets created with :func:`create_widget_for_node` will automatically
receive dispatched events:
* :data:`WorkflowEvent.InputLinkAdded` - when a new input link is added to
the workflow.
* :data:`LinkEvent.InputLinkRemoved` - when a input link is removed
* :data:`LinkEvent.OutputLinkAdded` - when a new output link is added to
the workflow
* :data:`LinkEvent.InputLinkRemoved` - when a output link is removed
* :data:`WorkflowEnvEvent.WorkflowEnvironmentChanged` - when the
workflow environment changes.
.. seealso:: :func:`.Scheme.add_link()`, :func:`Scheme.remove_link`,
:func:`.Scheme.runtime_env`
"""
#: A new QWidget was created and added by the manager.
widget_for_node_added = Signal(SchemeNode, QWidget)
#: A QWidget was removed, hidden and will be deleted when appropriate.
widget_for_node_removed = Signal(SchemeNode, QWidget)
class CreationPolicy(enum.Enum):
"""
Widget Creation Policy.
"""
#: Widgets are scheduled to be created from the event loop, or when
#: first accessed with `widget_for_node`
Normal = "Normal"
#: Widgets are created immediately when a node is added to the
#: workflow model.
Immediate = "Immediate"
#: Widgets are created only when first accessed with `widget_for_node`
#: (e.g. when activated in the view).
OnDemand = "OnDemand"
Normal = CreationPolicy.Normal
Immediate = CreationPolicy.Immediate
OnDemand = CreationPolicy.OnDemand
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__workflow = None # type: Optional[Scheme]
self.__creation_policy = WidgetManager.Normal
self.__float_widgets_on_top = False
self.__item_for_node = {} # type: Dict[SchemeNode, Item]
self.__item_for_widget = {} # type: Dict[QWidget, Item]
self.__init_queue = deque() # type: Deque[SchemeNode]
self.__init_timer = QTimer(self, singleShot=True)
self.__init_timer.timeout.connect(self.__process_init_queue)
self.__activation_monitor = ActivationMonitor(self)
self.__activation_counter = itertools.count()
self.__activation_monitor.activated.connect(self.__mark_activated)
def set_workflow(self, workflow):
# type: (Scheme) -> None
"""
Set the workflow.
"""
if workflow is self.__workflow:
return
if self.__workflow is not None:
# cleanup
for node in self.__workflow.nodes:
self.__remove_node(node)
self.__workflow.node_added.disconnect(self.__on_node_added)
self.__workflow.node_removed.disconnect(self.__on_node_removed)
self.__workflow.link_added.disconnect(self.__on_link_added)
self.__workflow.link_removed.disconnect(self.__on_link_removed)
self.__workflow.runtime_env_changed.disconnect(self.__on_env_changed)
self.__workflow.removeEventFilter(self)
self.__workflow = workflow
workflow.node_added.connect(
self.__on_node_added, Qt.UniqueConnection)
workflow.node_removed.connect(
self.__on_node_removed, Qt.UniqueConnection)
workflow.link_added.connect(
self.__on_link_added, Qt.UniqueConnection)
workflow.link_removed.connect(
self.__on_link_removed, Qt.UniqueConnection)
workflow.runtime_env_changed.connect(
self.__on_env_changed, Qt.UniqueConnection)
workflow.installEventFilter(self)
for node in workflow.nodes:
self.__add_node(node)
def workflow(self):
return self.__workflow
scheme = workflow
set_scheme = set_workflow
def set_creation_policy(self, policy):
# type: (CreationPolicy) -> None
"""
Set the widget creation policy.
"""
if self.__creation_policy != policy:
self.__creation_policy = policy
if self.__creation_policy == WidgetManager.Immediate:
self.__init_timer.stop()
# create all
if self.__workflow is not None:
for node in self.__workflow.nodes:
self.ensure_created(node)
elif self.__creation_policy == WidgetManager.Normal:
if not self.__init_timer.isActive() and self.__init_queue:
self.__init_timer.start()
elif self.__creation_policy == WidgetManager.OnDemand:
self.__init_timer.stop()
else:
assert False
def creation_policy(self):
"""
Return the current widget creation policy.
"""
return self.__creation_policy
def create_widget_for_node(self, node):
# type: (SchemeNode) -> QWidget
"""
Create and initialize a widget for node.
This is an abstract method. Subclasses must reimplemented it.
"""
raise NotImplementedError()
def delete_widget_for_node(self, node, widget):
"""
Remove and delete widget for node.
This is an abstract method. Subclasses must reimplemented it.
"""
raise NotImplementedError()
def node_for_widget(self, widget):
# type: (QWidget) -> Optional[SchemeNode]
"""
Return the node for widget.
"""
item = self.__item_for_widget.get(widget)
if item is not None:
return item.node
else:
return None
def widget_for_node(self, node):
# type: (SchemeNode) -> Optional[QWidget]
"""
Return the widget for node.
"""
self.ensure_created(node)
item = self.__item_for_node.get(node)
return item.widget if item is not None else None
def __add_widget_for_node(self, node):
# type: (SchemeNode) -> None
item = self.__item_for_node.get(node)
if item is not None:
return
if node not in self.__workflow.nodes:
return
if node in self.__init_queue:
self.__init_queue.remove(node)
item = Item(node, None, -1)
# Insert on the node -> item mapping.
self.__item_for_node[node] = item
log.debug("Creating widget for node %s", node)
try:
w = self.create_widget_for_node(node)
except Exception: # pylint: disable=broad-except
log.critical("", exc_info=True)
lines = traceback.format_exception(*sys.exc_info())
text = "".join(lines)
errorwidget = QLabel(
textInteractionFlags=Qt.TextSelectableByMouse, wordWrap=True,
objectName="widgetmanager-error-placeholder",
text="<pre>" + escape(text) + "</pre>"
)
item.errorwidget = errorwidget
node.set_state_message(UserMessage(text, UserMessage.Error, 0))
return
else:
item.widget = w
self.__item_for_widget[w] = item
self.__set_float_on_top_flag(w)
w.installEventFilter(self.__activation_monitor)
# Up shortcut (activate/open parent)
up_shortcut = QShortcut(
QKeySequence(Qt.ControlModifier + Qt.Key_Up), w)
up_shortcut.activated.connect(self.__on_activate_parent)
# send all the post creation notification events
workflow = self.__workflow
assert workflow is not None
inputs = workflow.find_links(sink_node=node)
for link in inputs:
ev = LinkEvent(LinkEvent.InputLinkAdded, link)
QCoreApplication.sendEvent(w, ev)
outputs = workflow.find_links(source_node=node)
for link in outputs:
ev = LinkEvent(LinkEvent.OutputLinkAdded, link)
QCoreApplication.sendEvent(w, ev)
self.widget_for_node_added.emit(node, w)
def ensure_created(self, node):
# type: (SchemeNode) -> None
"""
Ensure that the widget for node is created.
"""
if node not in self.__workflow.nodes:
return
item = self.__item_for_node.get(node)
if item is None:
self.__add_widget_for_node(node)
def __on_node_added(self, node): # type: (SchemeNode) -> None
assert self.__workflow is not None
assert node in self.__workflow.nodes
assert node not in self.__item_for_node
self.__add_node(node)
def __add_node(self, node): # type: (SchemeNode) -> None
# add node for tracking
node.installEventFilter(self)
if self.__creation_policy == WidgetManager.Immediate:
self.ensure_created(node)
elif self.__creation_policy == WidgetManager.Normal:
self.__init_queue.append(node)
self.__init_timer.start()
def __on_node_removed(self, node): # type: (SchemeNode) -> None
assert self.__workflow is not None
assert node not in self.__workflow.nodes
self.__remove_node(node)
def __remove_node(self, node): # type: (SchemeNode) -> None
# remove the node and its widget from tracking.
node.removeEventFilter(self)
if node in self.__init_queue:
self.__init_queue.remove(node)
item = self.__item_for_node.get(node)
if item is not None and item.widget is not None:
widget = item.widget
assert widget in self.__item_for_widget
del self.__item_for_widget[widget]
widget.removeEventFilter(self.__activation_monitor)
item.widget = None
self.widget_for_node_removed.emit(node, widget)
self.delete_widget_for_node(node, widget)
if item is not None:
del self.__item_for_node[node]
@Slot()
def __process_init_queue(self):
log.debug("__process_init_queue")
while self.__init_queue:
node = self.__init_queue.popleft()
assert node in self.__workflow.nodes
self.ensure_created(node)
def __on_link_added(self, link): # type: (SchemeLink) -> None
assert link.source_node in self.__workflow.nodes
assert link.sink_node in self.__workflow.nodes
source = self.__item_for_widget.get(link.source_node)
sink = self.__item_for_widget.get(link.sink_node)
# notify the node gui of an added link
if source is not None:
ev = LinkEvent(LinkEvent.OutputLinkAdded, link)
QCoreApplication.sendEvent(source.widget, ev)
if sink is not None:
ev = LinkEvent(LinkEvent.InputLinkAdded, link)
QCoreApplication.sendEvent(sink.widget, ev)
def __on_link_removed(self, link): # type: (SchemeLink) -> None
assert link.source_node in self.__workflow.nodes
assert link.sink_node in self.__workflow.nodes
source = self.__item_for_widget.get(link.source_node)
sink = self.__item_for_widget.get(link.sink_node)
# notify the node gui of an removed link
if source is not None:
ev = LinkEvent(LinkEvent.OutputLinkRemoved, link)
QCoreApplication.sendEvent(source.widget, ev)
if sink is not None:
ev = LinkEvent(LinkEvent.InputLinkRemoved, link)
QCoreApplication.sendEvent(sink.widget, ev)
def __mark_activated(self, widget): # type: (QWidget) -> None
# Update the tracked stacking order for `widget`
item = self.__item_for_widget.get(widget)
if item is not None:
item.activation_order = next(self.__activation_counter)
def activate_widget_for_node(self, node, widget):
# type: (SchemeNode, QWidget) -> None
"""
Activate the widget for node (show and raise above other)
"""
if widget.windowState() == Qt.WindowMinimized:
widget.showNormal()
widget.setVisible(True)
widget.raise_()
widget.activateWindow()
def activate_window_group(self, group):
# type: (Scheme.WindowGroup) -> None
self.restore_window_state(group.state)
def raise_widgets_to_front(self):
"""
Raise all current visible widgets to the front.
The widgets will be stacked by activation order.
"""
workflow = self.__workflow
if workflow is None:
return
items = filter(
lambda item: (
item.widget.isVisible()
if item is not None and item.widget is not None
else False)
,
map(self.__item_for_node.get, workflow.nodes))
self.__raise_and_activate(items)
def set_float_widgets_on_top(self, float_on_top):
"""
Set `Float Widgets on Top` flag on all widgets.
"""
self.__float_widgets_on_top = float_on_top
for item in self.__item_for_node.values():
if item.widget is not None:
self.__set_float_on_top_flag(item.widget)
def save_window_state(self):
# type: () -> List[Tuple[SchemeNode, bytes]]
"""
Save current open window arrangement.
"""
workflow = self.__workflow # type: Scheme
state = []
for node in workflow.nodes: # type: SchemeNode
item = self.__item_for_node.get(node, None)
if item is None:
continue
stackorder = item.activation_order
if item.widget is not None and not item.widget.isHidden():
data = self.save_widget_geometry(node, item.widget)
state.append((stackorder, node, data))
state = [(node, data)
for _, node, data in sorted(state, key=lambda t: t[0])]
return state
def restore_window_state(self, state):
# type: (List[Tuple[SchemeNode, bytes]]) -> None
"""
Restore the window state.
"""
workflow = self.__workflow # type: Scheme
visible = {node for node, _ in state}
# first hide all other widgets
for node in workflow.nodes:
if node not in visible:
# avoid creating widgets if not needed
item = self.__item_for_node.get(node, None)
if item is not None and item.widget is not None:
item.widget.hide()
allnodes = set(workflow.nodes)
# restore state for visible group; windows are stacked as they appear
# in the state list.
w = None
for node, state in filter(lambda t: t[0] in allnodes, state):
w = self.widget_for_node(node) # also create it if needed
if w is not None:
w.show()
self.restore_widget_geometry(node, w, state)
w.raise_()
self.__mark_activated(w)
# activate (give focus to) the last window
if w is not None:
w.activateWindow()
def save_widget_geometry(self, node, widget):
# type: (SchemeNode, QWidget) -> bytes
"""
Save and return the current geometry and state for node.
"""
return b''
def restore_widget_geometry(self, node, widget, state):
# type: (SchemeNode, QWidget, bytes) -> bool
"""
Restore the widget geometry and state for node.
Return True if the geometry was restored successfully.
The default implementation does nothing.
"""
return False
def __raise_and_activate(self, items):
# type: (Iterable[Item]) -> None
"""Show and raise a set of widgets."""
# preserve the tracked stacking order
items = sorted(items, key=lambda item: item.activation_order)
w = None
for item in items:
if item.widget is not None:
w = item.widget
elif item.errorwidget is not None:
w = item.errorwidget
else:
continue
w.show()
w.raise_()
if w is not None:
# give focus to the last activated top window
w.activateWindow()
def __activate_widget_for_node(self, node): # type: (SchemeNode) -> None
# activate the widget for the node.
self.ensure_created(node)
item = self.__item_for_node.get(node)
if item is None:
return
if item.widget is not None:
self.activate_widget_for_node(node, item.widget)
elif item.errorwidget is not None:
item.errorwidget.show()
item.errorwidget.raise_()
item.errorwidget.activateWindow()
def __on_activate_parent(self):
event = WorkflowEvent(WorkflowEvent.ActivateParentRequest)
QCoreApplication.sendEvent(self.scheme(), event)
def eventFilter(self, recv, event):
# type: (QObject, QEvent) -> bool
if event.type() == NodeEvent.NodeActivateRequest \
and isinstance(recv, SchemeNode):
self.__activate_widget_for_node(recv)
return False
def __set_float_on_top_flag(self, widget):
"""Set or unset widget's float on top flag"""
should_float_on_top = self.__float_widgets_on_top
float_on_top = bool(widget.windowFlags() & Qt.WindowStaysOnTopHint)
if float_on_top == should_float_on_top:
return
widget_was_visible = widget.isVisible()
if should_float_on_top:
widget.setWindowFlags(
widget.windowFlags() | Qt.WindowStaysOnTopHint)
else:
widget.setWindowFlags(
widget.windowFlags() & ~Qt.WindowStaysOnTopHint)
# Changing window flags hid the widget
if widget_was_visible:
widget.show()
def __on_env_changed(self, key, newvalue, oldvalue):
# Notify widgets of a runtime environment change
for item in self.__item_for_node.values():
if item.widget is not None:
ev = WorkflowEnvChanged(key, newvalue, oldvalue)
QCoreApplication.sendEvent(item.widget, ev)
def actions_for_context_menu(self, node):
# type: (SchemeNode) -> List[QAction]
"""
Return a list of extra actions that can be inserted into context
menu in the workflow editor.
Subclasses can reimplement this method to extend the default context
menu.
Parameters
----------
node: SchemeNode
The node for which the context menu is requested.
Return
------
actions: List[QAction]
Actions that are appended to the default menu.
"""
return []
class CanvasView(QGraphicsView):
"""Canvas View handles the zooming.
"""
def __init__(self, *args):
super().__init__(*args)
self.setAlignment(Qt.AlignTop | Qt.AlignLeft)
self.__backgroundIcon = QIcon()
self.__autoScroll = False
self.__autoScrollMargin = 16
self.__autoScrollTimer = QTimer(self)
self.__autoScrollTimer.timeout.connect(self.__autoScrollAdvance)
# scale factor accumulating partial increments from wheel events
self.__zoomLevel = 100
# effective scale level(rounded to whole integers)
self.__effectiveZoomLevel = 100
self.__zoomInAction = QAction(
self.tr("Zoom in"), self, objectName="action-zoom-in",
shortcut=QKeySequence.ZoomIn,
triggered=self.zoomIn,
)
self.__zoomOutAction = QAction(
self.tr("Zoom out"), self, objectName="action-zoom-out",
shortcut=QKeySequence.ZoomOut,
triggered=self.zoomOut
)
self.__zoomResetAction = QAction(
self.tr("Reset Zoom"), self, objectName="action-zoom-reset",
triggered=self.zoomReset,
shortcut=QKeySequence(Qt.ControlModifier | Qt.Key_0)
)
def setScene(self, scene):
super().setScene(scene)
self._ensureSceneRect(scene)
def _ensureSceneRect(self, scene):
r = scene.addRect(QRectF(0, 0, 400, 400))
scene.sceneRect()
scene.removeItem(r)
def setAutoScrollMargin(self, margin):
self.__autoScrollMargin = margin
def autoScrollMargin(self):
return self.__autoScrollMargin
def setAutoScroll(self, enable):
self.__autoScroll = enable
def autoScroll(self):
return self.__autoScroll
def mousePressEvent(self, event):
super().mousePressEvent(event)
def mouseMoveEvent(self, event):
if event.buttons() & Qt.LeftButton:
if not self.__autoScrollTimer.isActive() and \
self.__shouldAutoScroll(event.pos()):
self.__startAutoScroll()
super().mouseMoveEvent(event)
def mouseReleaseEvent(self, event):
if event.button() & Qt.LeftButton:
self.__stopAutoScroll()
return super().mouseReleaseEvent(event)
def wheelEvent(self, event: QWheelEvent):
if event.modifiers() & Qt.ControlModifier \
and event.buttons() == Qt.NoButton:
delta = event.angleDelta().y()
# use mouse position as anchor while zooming
anchor = self.transformationAnchor()
self.setTransformationAnchor(QGraphicsView.AnchorUnderMouse)
self.__setZoomLevel(self.__zoomLevel + 10 * delta / 120)
self.setTransformationAnchor(anchor)
event.accept()
else:
super().wheelEvent(event)
def zoomIn(self):
self.__setZoomLevel(self.__zoomLevel + 10)
def zoomOut(self):
self.__setZoomLevel(self.__zoomLevel - 10)
def zoomReset(self):
"""
Reset the zoom level.
"""
self.__setZoomLevel(100)
def zoomLevel(self):
# type: () -> float
"""
Return the current zoom level.
Level is expressed in percentages; 100 is unscaled, 50 is half size, ...
"""
return self.__effectiveZoomLevel
def setZoomLevel(self, level):
self.__setZoomLevel(level)
def __setZoomLevel(self, scale):
self.__zoomLevel = max(30, min(scale, 300))
scale = round(self.__zoomLevel)
self.__zoomOutAction.setEnabled(scale != 30)
self.__zoomInAction.setEnabled(scale != 300)
if self.__effectiveZoomLevel != scale:
self.__effectiveZoomLevel = scale
transform = QTransform()
transform.scale(scale / 100, scale / 100)
self.setTransform(transform)
self.zoomLevelChanged.emit(scale)
zoomLevelChanged = Signal(float)
zoomLevel_ = Property(
float, zoomLevel, setZoomLevel, notify=zoomLevelChanged
)
def __shouldAutoScroll(self, pos):
if self.__autoScroll:
margin = self.__autoScrollMargin
viewrect = self.contentsRect()
rect = viewrect.adjusted(margin, margin, -margin, -margin)
# only do auto scroll when on the viewport's margins
return not rect.contains(pos) and viewrect.contains(pos)
else:
return False
def __startAutoScroll(self):
self.__autoScrollTimer.start(10)
log.debug("Auto scroll timer started")
def __stopAutoScroll(self):
if self.__autoScrollTimer.isActive():
self.__autoScrollTimer.stop()
log.debug("Auto scroll timer stopped")
def __autoScrollAdvance(self):
"""Advance the auto scroll
"""
pos = QCursor.pos()
pos = self.mapFromGlobal(pos)
margin = self.__autoScrollMargin
vvalue = self.verticalScrollBar().value()
hvalue = self.horizontalScrollBar().value()
vrect = QRect(0, 0, self.width(), self.height())
# What should be the speed
advance = 10
# We only do auto scroll if the mouse is inside the view.
if vrect.contains(pos):
if pos.x() < vrect.left() + margin:
self.horizontalScrollBar().setValue(hvalue - advance)
if pos.y() < vrect.top() + margin:
self.verticalScrollBar().setValue(vvalue - advance)
if pos.x() > vrect.right() - margin:
self.horizontalScrollBar().setValue(hvalue + advance)
if pos.y() > vrect.bottom() - margin:
self.verticalScrollBar().setValue(vvalue + advance)
if self.verticalScrollBar().value() == vvalue and \
self.horizontalScrollBar().value() == hvalue:
self.__stopAutoScroll()
else:
self.__stopAutoScroll()
log.debug("Auto scroll advance")
def setBackgroundIcon(self, icon):
if not isinstance(icon, QIcon):
raise TypeError("A QIcon expected.")
if self.__backgroundIcon != icon:
self.__backgroundIcon = icon
self.viewport().update()
def backgroundIcon(self):
return QIcon(self.__backgroundIcon)
def drawBackground(self, painter, rect):
super().drawBackground(painter, rect)
if not self.__backgroundIcon.isNull():
painter.setClipRect(rect)
vrect = QRect(QPoint(0, 0), self.viewport().size())
vrect = self.mapToScene(vrect).boundingRect()
pm = self.__backgroundIcon.pixmap(
vrect.size().toSize().boundedTo(QSize(200, 200))
)
pmrect = QRect(QPoint(0, 0), pm.size())
pmrect.moveCenter(vrect.center().toPoint())
if rect.toRect().intersects(pmrect):
painter.drawPixmap(pmrect, pm)
class OWScatterPlot(OWDataProjectionWidget):
"""Scatterplot visualization with explorative analysis and intelligent
data visualization enhancements."""
name = 'Scatter Plot'
description = "Interactive scatter plot visualization with " \
"intelligent data visualization enhancements."
icon = "icons/ScatterPlot.svg"
priority = 140
keywords = []
class Inputs(OWDataProjectionWidget.Inputs):
features = Input("Features", AttributeList)
class Outputs(OWDataProjectionWidget.Outputs):
features = Output("Features", AttributeList, dynamic=False)
settings_version = 4
auto_sample = Setting(True)
attr_x = ContextSetting(None)
attr_y = ContextSetting(None)
tooltip_shows_all = Setting(True)
GRAPH_CLASS = OWScatterPlotGraph
graph = SettingProvider(OWScatterPlotGraph)
embedding_variables_names = None
xy_changed_manually = Signal(Variable, Variable)
class Warning(OWDataProjectionWidget.Warning):
missing_coords = Msg(
"Plot cannot be displayed because '{}' or '{}' "
"is missing for all data points")
no_continuous_vars = Msg("Data has no continuous variables")
class Information(OWDataProjectionWidget.Information):
sampled_sql = Msg("Large SQL table; showing a sample.")
missing_coords = Msg(
"Points with missing '{}' or '{}' are not displayed")
def __init__(self):
self.sql_data = None # Orange.data.sql.table.SqlTable
self.attribute_selection_list = None # list of Orange.data.Variable
self.__timer = QTimer(self, interval=1200)
self.__timer.timeout.connect(self.add_data)
super().__init__()
# manually register Matplotlib file writers
self.graph_writers = self.graph_writers.copy()
for w in [MatplotlibFormat, MatplotlibPDFFormat]:
for ext in w.EXTENSIONS:
self.graph_writers[ext] = w
def _add_controls(self):
self._add_controls_axis()
self._add_controls_sampling()
super()._add_controls()
self.gui.add_widgets(
[self.gui.ShowGridLines,
self.gui.ToolTipShowsAll,
self.gui.RegressionLine],
self._plot_box)
gui.checkBox(
gui.indentedBox(self._plot_box), self,
value="graph.orthonormal_regression",
label="Treat variables as independent",
callback=self.graph.update_regression_line,
tooltip=
"If checked, fit line to group (minimize distance from points);\n"
"otherwise fit y as a function of x (minimize vertical distances)")
def _add_controls_axis(self):
common_options = dict(
labelWidth=50, orientation=Qt.Horizontal, sendSelectedValue=True,
valueType=str, contentsLength=14
)
box = gui.vBox(self.controlArea, True)
dmod = DomainModel
self.xy_model = DomainModel(dmod.MIXED, valid_types=ContinuousVariable)
self.cb_attr_x = gui.comboBox(
box, self, "attr_x", label="Axis x:",
callback=self.set_attr_from_combo,
model=self.xy_model, **common_options)
self.cb_attr_y = gui.comboBox(
box, self, "attr_y", label="Axis y:",
callback=self.set_attr_from_combo,
model=self.xy_model, **common_options)
vizrank_box = gui.hBox(box)
self.vizrank, self.vizrank_button = ScatterPlotVizRank.add_vizrank(
vizrank_box, self, "Find Informative Projections", self.set_attr)
def _add_controls_sampling(self):
self.sampling = gui.auto_commit(
self.controlArea, self, "auto_sample", "Sample", box="Sampling",
callback=self.switch_sampling, commit=lambda: self.add_data(1))
self.sampling.setVisible(False)
@property
def effective_variables(self):
return [self.attr_x, self.attr_y]
def _vizrank_color_change(self):
self.vizrank.initialize()
is_enabled = self.data is not None and not self.data.is_sparse() and \
len(self.xy_model) > 2 and len(self.data[self.valid_data]) > 1 \
and np.all(np.nan_to_num(np.nanstd(self.data.X, 0)) != 0)
self.vizrank_button.setEnabled(
is_enabled and self.attr_color is not None and
not np.isnan(self.data.get_column_view(
self.attr_color)[0].astype(float)).all())
text = "Color variable has to be selected." \
if is_enabled and self.attr_color is None else ""
self.vizrank_button.setToolTip(text)
def set_data(self, data):
if self.data and data and self.data.checksum() == data.checksum():
return
super().set_data(data)
def findvar(name, iterable):
"""Find a Orange.data.Variable in `iterable` by name"""
for el in iterable:
if isinstance(el, Variable) and el.name == name:
return el
return None
# handle restored settings from < 3.3.9 when attr_* were stored
# by name
if isinstance(self.attr_x, str):
self.attr_x = findvar(self.attr_x, self.xy_model)
if isinstance(self.attr_y, str):
self.attr_y = findvar(self.attr_y, self.xy_model)
if isinstance(self.attr_label, str):
self.attr_label = findvar(self.attr_label, self.gui.label_model)
if isinstance(self.attr_color, str):
self.attr_color = findvar(self.attr_color, self.gui.color_model)
if isinstance(self.attr_shape, str):
self.attr_shape = findvar(self.attr_shape, self.gui.shape_model)
if isinstance(self.attr_size, str):
self.attr_size = findvar(self.attr_size, self.gui.size_model)
def check_data(self):
self.clear_messages()
self.__timer.stop()
self.sampling.setVisible(False)
self.sql_data = None
if isinstance(self.data, SqlTable):
if self.data.approx_len() < 4000:
self.data = Table(self.data)
else:
self.Information.sampled_sql()
self.sql_data = self.data
data_sample = self.data.sample_time(0.8, no_cache=True)
data_sample.download_data(2000, partial=True)
self.data = Table(data_sample)
self.sampling.setVisible(True)
if self.auto_sample:
self.__timer.start()
if self.data is not None:
if not self.data.domain.has_continuous_attributes(True, True):
self.Warning.no_continuous_vars()
self.data = None
if self.data is not None and (len(self.data) == 0 or
len(self.data.domain) == 0):
self.data = None
def get_embedding(self):
self.valid_data = None
if self.data is None:
return None
x_data = self.get_column(self.attr_x, filter_valid=False)
y_data = self.get_column(self.attr_y, filter_valid=False)
if x_data is None or y_data is None:
return None
self.Warning.missing_coords.clear()
self.Information.missing_coords.clear()
self.valid_data = np.isfinite(x_data) & np.isfinite(y_data)
if self.valid_data is not None and not np.all(self.valid_data):
msg = self.Information if np.any(self.valid_data) else self.Warning
msg.missing_coords(self.attr_x.name, self.attr_y.name)
return np.vstack((x_data, y_data)).T
# Tooltip
def _point_tooltip(self, point_id, skip_attrs=()):
point_data = self.data[point_id]
xy_attrs = (self.attr_x, self.attr_y)
text = "<br/>".join(
escape('{} = {}'.format(var.name, point_data[var]))
for var in xy_attrs)
if self.tooltip_shows_all:
others = super()._point_tooltip(point_id, skip_attrs=xy_attrs)
if others:
text = "<b>{}</b><br/><br/>{}".format(text, others)
return text
def add_data(self, time=0.4):
if self.data and len(self.data) > 2000:
self.__timer.stop()
return
data_sample = self.sql_data.sample_time(time, no_cache=True)
if data_sample:
data_sample.download_data(2000, partial=True)
data = Table(data_sample)
self.data = Table.concatenate((self.data, data), axis=0)
self.handleNewSignals()
def init_attr_values(self):
super().init_attr_values()
data = self.data
domain = data.domain if data and len(data) else None
self.xy_model.set_domain(domain)
self.attr_x = self.xy_model[0] if self.xy_model else None
self.attr_y = self.xy_model[1] if len(self.xy_model) >= 2 \
else self.attr_x
def switch_sampling(self):
self.__timer.stop()
if self.auto_sample and self.sql_data:
self.add_data()
self.__timer.start()
def set_subset_data(self, subset_data):
self.warning()
if isinstance(subset_data, SqlTable):
if subset_data.approx_len() < AUTO_DL_LIMIT:
subset_data = Table(subset_data)
else:
self.warning("Data subset does not support large Sql tables")
subset_data = None
super().set_subset_data(subset_data)
# called when all signals are received, so the graph is updated only once
def handleNewSignals(self):
if self.attribute_selection_list and self.data is not None and \
self.data.domain is not None and \
all(attr in self.data.domain for attr
in self.attribute_selection_list):
self.attr_x, self.attr_y = self.attribute_selection_list[:2]
self.attribute_selection_list = None
super().handleNewSignals()
self._vizrank_color_change()
@Inputs.features
def set_shown_attributes(self, attributes):
if attributes and len(attributes) >= 2:
self.attribute_selection_list = attributes[:2]
self._invalidated = self._invalidated \
or self.attr_x != attributes[0] \
or self.attr_y != attributes[1]
else:
self.attribute_selection_list = None
def set_attr(self, attr_x, attr_y):
if attr_x != self.attr_x or attr_y != self.attr_y:
self.attr_x, self.attr_y = attr_x, attr_y
self.attr_changed()
def set_attr_from_combo(self):
self.attr_changed()
self.xy_changed_manually.emit(self.attr_x, self.attr_y)
def attr_changed(self):
self.setup_plot()
self.commit()
def get_axes(self):
return {"bottom": self.attr_x, "left": self.attr_y}
def colors_changed(self):
super().colors_changed()
self._vizrank_color_change()
def commit(self):
super().commit()
self.send_features()
def send_features(self):
features = [attr for attr in [self.attr_x, self.attr_y] if attr]
self.Outputs.features.send(features or None)
def get_widget_name_extension(self):
if self.data is not None:
return "{} vs {}".format(self.attr_x.name, self.attr_y.name)
return None
@classmethod
def migrate_settings(cls, settings, version):
if version < 2 and "selection" in settings and settings["selection"]:
settings["selection_group"] = [(a, 1) for a in settings["selection"]]
if version < 3:
if "auto_send_selection" in settings:
settings["auto_commit"] = settings["auto_send_selection"]
if "selection_group" in settings:
settings["selection"] = settings["selection_group"]
@classmethod
def migrate_context(cls, context, version):
values = context.values
if version < 3:
values["attr_color"] = values["graph"]["attr_color"]
values["attr_size"] = values["graph"]["attr_size"]
values["attr_shape"] = values["graph"]["attr_shape"]
values["attr_label"] = values["graph"]["attr_label"]
if version < 4:
if values["attr_x"][1] % 100 == 1 or values["attr_y"][1] % 100 == 1:
raise IncompatibleContext()
class CanvasView(QGraphicsView):
"""Canvas View handles the zooming.
"""
def __init__(self, *args):
super().__init__(*args)
self.setAlignment(Qt.AlignTop | Qt.AlignLeft)
self.__backgroundIcon = QIcon()
self.__autoScroll = False
self.__autoScrollMargin = 16
self.__autoScrollTimer = QTimer(self)
self.__autoScrollTimer.timeout.connect(self.__autoScrollAdvance)
# scale factor accumulating partial increments from wheel events
self.__zoomLevel = 100
# effective scale level(rounded to whole integers)
self.__effectiveZoomLevel = 100
self.__zoomInAction = QAction(
self.tr("Zoom in"),
self,
objectName="action-zoom-in",
shortcut=QKeySequence.ZoomIn,
triggered=self.zoomIn,
)
self.__zoomOutAction = QAction(self.tr("Zoom out"),
self,
objectName="action-zoom-out",
shortcut=QKeySequence.ZoomOut,
triggered=self.zoomOut)
self.__zoomResetAction = QAction(
self.tr("Reset Zoom"),
self,
objectName="action-zoom-reset",
triggered=self.zoomReset,
shortcut=QKeySequence(Qt.ControlModifier | Qt.Key_0))
def setScene(self, scene):
super().setScene(scene)
self._ensureSceneRect(scene)
def _ensureSceneRect(self, scene):
r = scene.addRect(QRectF(0, 0, 400, 400))
scene.sceneRect()
scene.removeItem(r)
def setAutoScrollMargin(self, margin):
self.__autoScrollMargin = margin
def autoScrollMargin(self):
return self.__autoScrollMargin
def setAutoScroll(self, enable):
self.__autoScroll = enable
def autoScroll(self):
return self.__autoScroll
def mousePressEvent(self, event):
super().mousePressEvent(event)
def mouseMoveEvent(self, event):
if event.buttons() & Qt.LeftButton:
if not self.__autoScrollTimer.isActive() and \
self.__shouldAutoScroll(event.pos()):
self.__startAutoScroll()
super().mouseMoveEvent(event)
def mouseReleaseEvent(self, event):
if event.button() & Qt.LeftButton:
self.__stopAutoScroll()
return super().mouseReleaseEvent(event)
def wheelEvent(self, event: QWheelEvent):
if event.modifiers() & Qt.ControlModifier \
and event.buttons() == Qt.NoButton:
delta = event.angleDelta().y()
# use mouse position as anchor while zooming
anchor = self.transformationAnchor()
self.setTransformationAnchor(QGraphicsView.AnchorUnderMouse)
self.__setZoomLevel(self.__zoomLevel + 10 * delta / 120)
self.setTransformationAnchor(anchor)
event.accept()
else:
super().wheelEvent(event)
def zoomIn(self):
self.__setZoomLevel(self.__zoomLevel + 10)
def zoomOut(self):
self.__setZoomLevel(self.__zoomLevel - 10)
def zoomReset(self):
"""
Reset the zoom level.
"""
self.__setZoomLevel(100)
def zoomLevel(self):
# type: () -> float
"""
Return the current zoom level.
Level is expressed in percentages; 100 is unscaled, 50 is half size, ...
"""
return self.__effectiveZoomLevel
def setZoomLevel(self, level):
self.__setZoomLevel(level)
def __setZoomLevel(self, scale):
# type: (float) -> None
self.__zoomLevel = max(30, min(scale, 300))
scale = round(self.__zoomLevel)
self.__zoomOutAction.setEnabled(scale != 30)
self.__zoomInAction.setEnabled(scale != 300)
if self.__effectiveZoomLevel != scale:
self.__effectiveZoomLevel = scale
transform = QTransform()
transform.scale(scale / 100, scale / 100)
self.setTransform(transform)
self.zoomLevelChanged.emit(scale)
zoomLevelChanged = Signal(float)
zoomLevel_ = Property(float,
zoomLevel,
setZoomLevel,
notify=zoomLevelChanged)
def __shouldAutoScroll(self, pos):
if self.__autoScroll:
margin = self.__autoScrollMargin
viewrect = self.contentsRect()
rect = viewrect.adjusted(margin, margin, -margin, -margin)
# only do auto scroll when on the viewport's margins
return not rect.contains(pos) and viewrect.contains(pos)
else:
return False
def __startAutoScroll(self):
self.__autoScrollTimer.start(10)
log.debug("Auto scroll timer started")
def __stopAutoScroll(self):
if self.__autoScrollTimer.isActive():
self.__autoScrollTimer.stop()
log.debug("Auto scroll timer stopped")
def __autoScrollAdvance(self):
"""Advance the auto scroll
"""
pos = QCursor.pos()
pos = self.mapFromGlobal(pos)
margin = self.__autoScrollMargin
vvalue = self.verticalScrollBar().value()
hvalue = self.horizontalScrollBar().value()
vrect = QRect(0, 0, self.width(), self.height())
# What should be the speed
advance = 10
# We only do auto scroll if the mouse is inside the view.
if vrect.contains(pos):
if pos.x() < vrect.left() + margin:
self.horizontalScrollBar().setValue(hvalue - advance)
if pos.y() < vrect.top() + margin:
self.verticalScrollBar().setValue(vvalue - advance)
if pos.x() > vrect.right() - margin:
self.horizontalScrollBar().setValue(hvalue + advance)
if pos.y() > vrect.bottom() - margin:
self.verticalScrollBar().setValue(vvalue + advance)
if self.verticalScrollBar().value() == vvalue and \
self.horizontalScrollBar().value() == hvalue:
self.__stopAutoScroll()
else:
self.__stopAutoScroll()
log.debug("Auto scroll advance")
def setBackgroundIcon(self, icon):
if not isinstance(icon, QIcon):
raise TypeError("A QIcon expected.")
if self.__backgroundIcon != icon:
self.__backgroundIcon = icon
self.viewport().update()
def backgroundIcon(self):
return QIcon(self.__backgroundIcon)
def drawBackground(self, painter, rect):
super().drawBackground(painter, rect)
if not self.__backgroundIcon.isNull():
painter.setClipRect(rect)
vrect = QRect(QPoint(0, 0), self.viewport().size())
vrect = self.mapToScene(vrect).boundingRect()
pm = self.__backgroundIcon.pixmap(vrect.size().toSize().boundedTo(
QSize(200, 200)))
pmrect = QRect(QPoint(0, 0), pm.size())
pmrect.moveCenter(vrect.center().toPoint())
if rect.toRect().intersects(pmrect):
painter.drawPixmap(pmrect, pm)
class OWScatterPlot(OWWidget):
"""Scatterplot visualization with explorative analysis and intelligent
data visualization enhancements."""
name = 'Scatter Plot'
description = "Interactive scatter plot visualization with " \
"intelligent data visualization enhancements."
icon = "icons/ScatterPlot.svg"
priority = 140
class Inputs:
data = Input("Data", Table, default=True)
data_subset = Input("Data Subset", Table)
features = Input("Features", AttributeList)
class Outputs:
selected_data = Output("Selected Data", Table, default=True)
annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME, Table)
features = Output("Features", Table, dynamic=False)
settingsHandler = DomainContextHandler()
auto_send_selection = Setting(True)
auto_sample = Setting(True)
toolbar_selection = Setting(0)
attr_x = ContextSetting(None)
attr_y = ContextSetting(None)
selection = Setting(None, schema_only=True)
graph = SettingProvider(OWScatterPlotGraph)
jitter_sizes = [0, 0.1, 0.5, 1, 2, 3, 4, 5, 7, 10]
graph_name = "graph.plot_widget.plotItem"
class Information(OWWidget.Information):
sampled_sql = Msg("Large SQL table; showing a sample.")
def __init__(self):
super().__init__()
box = gui.vBox(self.mainArea, True, margin=0)
self.graph = OWScatterPlotGraph(self, box, "ScatterPlot")
box.layout().addWidget(self.graph.plot_widget)
plot = self.graph.plot_widget
axispen = QPen(self.palette().color(QPalette.Text))
axis = plot.getAxis("bottom")
axis.setPen(axispen)
axis = plot.getAxis("left")
axis.setPen(axispen)
self.data = None # Orange.data.Table
self.subset_data = None # Orange.data.Table
self.data_metas_X = None # self.data, where primitive metas are moved to X
self.sql_data = None # Orange.data.sql.table.SqlTable
self.attribute_selection_list = None # list of Orange.data.Variable
self.__timer = QTimer(self, interval=1200)
self.__timer.timeout.connect(self.add_data)
common_options = dict(
labelWidth=50, orientation=Qt.Horizontal, sendSelectedValue=True,
valueType=str)
box = gui.vBox(self.controlArea, "Axis Data")
dmod = DomainModel
self.xy_model = DomainModel(dmod.MIXED, valid_types=dmod.PRIMITIVE)
self.cb_attr_x = gui.comboBox(
box, self, "attr_x", label="Axis x:", callback=self.update_attr,
model=self.xy_model, **common_options)
self.cb_attr_y = gui.comboBox(
box, self, "attr_y", label="Axis y:", callback=self.update_attr,
model=self.xy_model, **common_options)
vizrank_box = gui.hBox(box)
gui.separator(vizrank_box, width=common_options["labelWidth"])
self.vizrank, self.vizrank_button = ScatterPlotVizRank.add_vizrank(
vizrank_box, self, "Find Informative Projections", self.set_attr)
gui.separator(box)
gui.valueSlider(
box, self, value='graph.jitter_size', label='Jittering: ',
values=self.jitter_sizes, callback=self.reset_graph_data,
labelFormat=lambda x:
"None" if x == 0 else ("%.1f %%" if x < 1 else "%d %%") % x)
gui.checkBox(
gui.indentedBox(box), self, 'graph.jitter_continuous',
'Jitter numeric values', callback=self.reset_graph_data)
self.sampling = gui.auto_commit(
self.controlArea, self, "auto_sample", "Sample", box="Sampling",
callback=self.switch_sampling, commit=lambda: self.add_data(1))
self.sampling.setVisible(False)
g = self.graph.gui
g.point_properties_box(self.controlArea)
self.models = [self.xy_model] + g.points_models
box = gui.vBox(self.controlArea, "Plot Properties")
g.add_widgets([g.ShowLegend, g.ShowGridLines], box)
gui.checkBox(
box, self, value='graph.tooltip_shows_all',
label='Show all data on mouse hover')
self.cb_class_density = gui.checkBox(
box, self, value='graph.class_density', label='Show class density',
callback=self.update_density)
self.cb_reg_line = gui.checkBox(
box, self, value='graph.show_reg_line',
label='Show regression line', callback=self.update_regression_line)
gui.checkBox(
box, self, 'graph.label_only_selected',
'Label only selected points', callback=self.graph.update_labels)
self.zoom_select_toolbar = g.zoom_select_toolbar(
gui.vBox(self.controlArea, "Zoom/Select"), nomargin=True,
buttons=[g.StateButtonsBegin, g.SimpleSelect, g.Pan, g.Zoom,
g.StateButtonsEnd, g.ZoomReset]
)
buttons = self.zoom_select_toolbar.buttons
buttons[g.Zoom].clicked.connect(self.graph.zoom_button_clicked)
buttons[g.Pan].clicked.connect(self.graph.pan_button_clicked)
buttons[g.SimpleSelect].clicked.connect(self.graph.select_button_clicked)
buttons[g.ZoomReset].clicked.connect(self.graph.reset_button_clicked)
self.controlArea.layout().addStretch(100)
self.icons = gui.attributeIconDict
p = self.graph.plot_widget.palette()
self.graph.set_palette(p)
gui.auto_commit(self.controlArea, self, "auto_send_selection",
"Send Selection", "Send Automatically")
def zoom(s):
"""Zoom in/out by factor `s`."""
viewbox = plot.getViewBox()
# scaleBy scales the view's bounds (the axis range)
viewbox.scaleBy((1 / s, 1 / s))
def fit_to_view():
viewbox = plot.getViewBox()
viewbox.autoRange()
zoom_in = QAction(
"Zoom in", self, triggered=lambda: zoom(1.25)
)
zoom_in.setShortcuts([QKeySequence(QKeySequence.ZoomIn),
QKeySequence(self.tr("Ctrl+="))])
zoom_out = QAction(
"Zoom out", self, shortcut=QKeySequence.ZoomOut,
triggered=lambda: zoom(1 / 1.25)
)
zoom_fit = QAction(
"Fit in view", self,
shortcut=QKeySequence(Qt.ControlModifier | Qt.Key_0),
triggered=fit_to_view
)
self.addActions([zoom_in, zoom_out, zoom_fit])
def keyPressEvent(self, event):
super().keyPressEvent(event)
self.graph.update_tooltip(event.modifiers())
def keyReleaseEvent(self, event):
super().keyReleaseEvent(event)
self.graph.update_tooltip(event.modifiers())
# def settingsFromWidgetCallback(self, handler, context):
# context.selectionPolygons = []
# for curve in self.graph.selectionCurveList:
# xs = [curve.x(i) for i in range(curve.dataSize())]
# ys = [curve.y(i) for i in range(curve.dataSize())]
# context.selectionPolygons.append((xs, ys))
# def settingsToWidgetCallback(self, handler, context):
# selections = getattr(context, "selectionPolygons", [])
# for (xs, ys) in selections:
# c = SelectionCurve("")
# c.setData(xs,ys)
# c.attach(self.graph)
# self.graph.selectionCurveList.append(c)
def reset_graph_data(self, *_):
if self.data is not None:
self.graph.rescale_data()
self.update_graph()
@Inputs.data
def set_data(self, data):
self.clear_messages()
self.Information.sampled_sql.clear()
self.__timer.stop()
self.sampling.setVisible(False)
self.sql_data = None
if isinstance(data, SqlTable):
if data.approx_len() < 4000:
data = Table(data)
else:
self.Information.sampled_sql()
self.sql_data = data
data_sample = data.sample_time(0.8, no_cache=True)
data_sample.download_data(2000, partial=True)
data = Table(data_sample)
self.sampling.setVisible(True)
if self.auto_sample:
self.__timer.start()
if data is not None and (len(data) == 0 or len(data.domain) == 0):
data = None
if self.data and data and self.data.checksum() == data.checksum():
return
self.closeContext()
same_domain = (self.data and data and
data.domain.checksum() == self.data.domain.checksum())
self.data = data
self.data_metas_X = self.move_primitive_metas_to_X(data)
if not same_domain:
self.init_attr_values()
self.vizrank.initialize()
self.vizrank.attrs = self.data.domain.attributes if self.data is not None else []
self.vizrank_button.setEnabled(
self.data is not None and not self.data.is_sparse() and
self.data.domain.class_var is not None and
len(self.data.domain.attributes) > 1 and len(self.data) > 1)
if self.data is not None and self.data.domain.class_var is None \
and len(self.data.domain.attributes) > 1 and len(self.data) > 1:
self.vizrank_button.setToolTip(
"Data with a class variable is required.")
else:
self.vizrank_button.setToolTip("")
self.openContext(self.data)
def findvar(name, iterable):
"""Find a Orange.data.Variable in `iterable` by name"""
for el in iterable:
if isinstance(el, Orange.data.Variable) and el.name == name:
return el
return None
# handle restored settings from < 3.3.9 when attr_* were stored
# by name
if isinstance(self.attr_x, str):
self.attr_x = findvar(self.attr_x, self.xy_model)
if isinstance(self.attr_y, str):
self.attr_y = findvar(self.attr_y, self.xy_model)
if isinstance(self.graph.attr_label, str):
self.graph.attr_label = findvar(
self.graph.attr_label, self.graph.gui.label_model)
if isinstance(self.graph.attr_color, str):
self.graph.attr_color = findvar(
self.graph.attr_color, self.graph.gui.color_model)
if isinstance(self.graph.attr_shape, str):
self.graph.attr_shape = findvar(
self.graph.attr_shape, self.graph.gui.shape_model)
if isinstance(self.graph.attr_size, str):
self.graph.attr_size = findvar(
self.graph.attr_size, self.graph.gui.size_model)
def add_data(self, time=0.4):
if self.data and len(self.data) > 2000:
return self.__timer.stop()
data_sample = self.sql_data.sample_time(time, no_cache=True)
if data_sample:
data_sample.download_data(2000, partial=True)
data = Table(data_sample)
self.data = Table.concatenate((self.data, data), axis=0)
self.data_metas_X = self.move_primitive_metas_to_X(self.data)
self.handleNewSignals()
def switch_sampling(self):
self.__timer.stop()
if self.auto_sample and self.sql_data:
self.add_data()
self.__timer.start()
def move_primitive_metas_to_X(self, data):
if data is not None:
new_attrs = [a for a in data.domain.attributes + data.domain.metas
if a.is_primitive()]
new_metas = [m for m in data.domain.metas if not m.is_primitive()]
new_domain = Domain(new_attrs, data.domain.class_vars, new_metas)
data = data.transform(new_domain)
return data
@Inputs.data_subset
def set_subset_data(self, subset_data):
self.warning()
if isinstance(subset_data, SqlTable):
if subset_data.approx_len() < AUTO_DL_LIMIT:
subset_data = Table(subset_data)
else:
self.warning("Data subset does not support large Sql tables")
subset_data = None
self.subset_data = self.move_primitive_metas_to_X(subset_data)
self.controls.graph.alpha_value.setEnabled(subset_data is None)
# called when all signals are received, so the graph is updated only once
def handleNewSignals(self):
self.graph.new_data(self.sparse_to_dense(self.data_metas_X),
self.sparse_to_dense(self.subset_data))
if self.attribute_selection_list and self.graph.domain and \
all(attr in self.graph.domain
for attr in self.attribute_selection_list):
self.attr_x = self.attribute_selection_list[0]
self.attr_y = self.attribute_selection_list[1]
self.attribute_selection_list = None
self.update_graph()
self.cb_class_density.setEnabled(self.graph.can_draw_density())
self.cb_reg_line.setEnabled(self.graph.can_draw_regresssion_line())
self.apply_selection()
self.unconditional_commit()
def prepare_data(self):
"""
Only when dealing with sparse matrices.
GH-2152
"""
self.graph.new_data(self.sparse_to_dense(self.data_metas_X),
self.sparse_to_dense(self.subset_data),
new=False)
def sparse_to_dense(self, input_data=None):
if input_data is None or not input_data.is_sparse():
return input_data
keys = []
attrs = {self.attr_x,
self.attr_y,
self.graph.attr_color,
self.graph.attr_shape,
self.graph.attr_size,
self.graph.attr_label}
for i, attr in enumerate(input_data.domain):
if attr in attrs:
keys.append(i)
new_domain = input_data.domain.select_columns(keys)
dmx = input_data.transform(new_domain)
dmx.X = dmx.X.toarray()
# TODO: remove once we make sure Y is always dense.
if sp.issparse(dmx.Y):
dmx.Y = dmx.Y.toarray()
return dmx
def apply_selection(self):
"""Apply selection saved in workflow."""
if self.data is not None and self.selection is not None:
self.graph.selection = np.zeros(len(self.data), dtype=np.uint8)
self.selection = [x for x in self.selection if x < len(self.data)]
self.graph.selection[self.selection] = 1
self.graph.update_colors(keep_colors=True)
@Inputs.features
def set_shown_attributes(self, attributes):
if attributes and len(attributes) >= 2:
self.attribute_selection_list = attributes[:2]
else:
self.attribute_selection_list = None
def get_shown_attributes(self):
return self.attr_x, self.attr_y
def init_attr_values(self):
domain = self.data and self.data.domain
for model in self.models:
model.set_domain(domain)
self.attr_x = self.xy_model[0] if self.xy_model else None
self.attr_y = self.xy_model[1] if len(self.xy_model) >= 2 \
else self.attr_x
self.graph.attr_color = domain and self.data.domain.class_var or None
self.graph.attr_shape = None
self.graph.attr_size = None
self.graph.attr_label = None
def set_attr(self, attr_x, attr_y):
self.attr_x, self.attr_y = attr_x, attr_y
self.update_attr()
def update_attr(self):
self.prepare_data()
self.update_graph()
self.cb_class_density.setEnabled(self.graph.can_draw_density())
self.cb_reg_line.setEnabled(self.graph.can_draw_regresssion_line())
self.send_features()
def update_colors(self):
self.prepare_data()
self.cb_class_density.setEnabled(self.graph.can_draw_density())
def update_density(self):
self.update_graph(reset_view=False)
def update_regression_line(self):
self.update_graph(reset_view=False)
def update_graph(self, reset_view=True, **_):
self.graph.zoomStack = []
if self.graph.data is None:
return
self.graph.update_data(self.attr_x, self.attr_y, reset_view)
def selection_changed(self):
self.send_data()
@staticmethod
def create_groups_table(data, selection):
if data is None:
return None
names = [var.name for var in data.domain.variables + data.domain.metas]
name = get_next_name(names, "Selection group")
metas = data.domain.metas + (
DiscreteVariable(
name,
["Unselected"] + ["G{}".format(i + 1)
for i in range(np.max(selection))]),
)
domain = Domain(data.domain.attributes, data.domain.class_vars, metas)
table = data.transform(domain)
table.metas[:, len(data.domain.metas):] = \
selection.reshape(len(data), 1)
return table
def send_data(self):
selected = None
selection = None
# TODO: Implement selection for sql data
graph = self.graph
if isinstance(self.data, SqlTable):
selected = self.data
elif self.data is not None:
selection = graph.get_selection()
if len(selection) > 0:
selected = self.data[selection]
if graph.selection is not None and np.max(graph.selection) > 1:
annotated = self.create_groups_table(self.data, graph.selection)
else:
annotated = create_annotated_table(self.data, selection)
self.Outputs.selected_data.send(selected)
self.Outputs.annotated_data.send(annotated)
# Store current selection in a setting that is stored in workflow
if self.selection is not None and len(selection):
self.selection = list(selection)
def send_features(self):
features = None
if self.attr_x or self.attr_y:
dom = Domain([], metas=(StringVariable(name="feature"),))
features = Table(dom, [[self.attr_x], [self.attr_y]])
features.name = "Features"
self.Outputs.features.send(features)
def commit(self):
self.send_data()
self.send_features()
def get_widget_name_extension(self):
if self.data is not None:
return "{} vs {}".format(self.attr_x.name, self.attr_y.name)
def send_report(self):
if self.data is None:
return
def name(var):
return var and var.name
caption = report.render_items_vert((
("Color", name(self.graph.attr_color)),
("Label", name(self.graph.attr_label)),
("Shape", name(self.graph.attr_shape)),
("Size", name(self.graph.attr_size)),
("Jittering", (self.attr_x.is_discrete or
self.attr_y.is_discrete or
self.graph.jitter_continuous) and
self.graph.jitter_size)))
self.report_plot()
if caption:
self.report_caption(caption)
def onDeleteWidget(self):
super().onDeleteWidget()
self.graph.plot_widget.getViewBox().deleteLater()
self.graph.plot_widget.clear()
class WidgetManager(QObject):
"""
OWWidget instance manager class.
This class handles the lifetime of OWWidget instances in a
:class:`WidgetsScheme`.
"""
#: A new OWWidget was created and added by the manager.
widget_for_node_added = Signal(SchemeNode, QWidget)
#: An OWWidget was removed, hidden and will be deleted when appropriate.
widget_for_node_removed = Signal(SchemeNode, QWidget)
class ProcessingState(enum.IntEnum):
"""Widget processing state flags"""
#: Signal manager is updating/setting the widget's inputs
InputUpdate = 1
#: Widget has entered a blocking state (OWWidget.isBlocking)
BlockingUpdate = 2
#: Widget has entered processing state
ProcessingUpdate = 4
#: Widget is still in the process of initialization
Initializing = 8
InputUpdate, BlockingUpdate, ProcessingUpdate, Initializing = ProcessingState
#: State mask for widgets that cannot be deleted immediately
#: (see __try_delete)
_DelayDeleteMask = InputUpdate | BlockingUpdate
#: Widget initialization states
Delayed = namedtuple("Delayed", ["node"])
PartiallyInitialized = namedtuple("Materializing",
["node", "partially_initialized_widget"])
Materialized = namedtuple("Materialized", ["node", "widget"])
class CreationPolicy(enum.Enum):
"""Widget Creation Policy"""
#: Widgets are scheduled to be created from the event loop, or when
#: first accessed with `widget_for_node`
Normal = "Normal"
#: Widgets are created immediately when added to the workflow model
Immediate = "Immediate"
#: Widgets are created only when first accessed with `widget_for_node`
OnDemand = "OnDemand"
Normal, Immediate, OnDemand = CreationPolicy
def __init__(self, parent=None):
QObject.__init__(self, parent)
self.__scheme = None
self.__signal_manager = None
self.__widgets = []
self.__initstate_for_node = {}
self.__creation_policy = WidgetManager.Normal
#: a queue of all nodes whose widgets are scheduled for
#: creation/initialization
self.__init_queue = deque() # type: Deque[SchemeNode]
#: Timer for scheduling widget initialization
self.__init_timer = QTimer(self, interval=0, singleShot=True)
self.__init_timer.timeout.connect(self.__create_delayed)
#: A mapping of SchemeNode -> OWWidget (note: a mapping is only added
#: after the widget is actually created)
self.__widget_for_node = {}
#: a mapping of OWWidget -> SchemeNode
self.__node_for_widget = {}
# Widgets that were 'removed' from the scheme but were at
# the time in an input update loop and could not be deleted
# immediately
self.__delay_delete = set()
#: processing state flags for all widgets (including the ones
#: in __delay_delete).
#: Note: widgets which have not yet been created do not have an entry
self.__widget_processing_state = {}
# Tracks the widget in the update loop by the SignalManager
self.__updating_widget = None
# Widgets float above other windows
self.__float_widgets_on_top = False
if hasattr(qApp, "applicationStateChanged"):
# disables/enables widget floating when app (de)activates
# available in Qt >= 5.2
def reapply_float_on_top():
self.set_float_widgets_on_top(self.__float_widgets_on_top)
qApp.applicationStateChanged.connect(reapply_float_on_top)
def set_scheme(self, scheme):
"""
Set the :class:`WidgetsScheme` instance to manage.
"""
self.__scheme = scheme
self.__signal_manager = scheme.findChild(SignalManager)
self.__signal_manager.processingStarted[SchemeNode].connect(
self.__on_processing_started)
self.__signal_manager.processingFinished[SchemeNode].connect(
self.__on_processing_finished)
scheme.node_added.connect(self.add_widget_for_node)
scheme.node_removed.connect(self.remove_widget_for_node)
scheme.runtime_env_changed.connect(self.__on_env_changed)
scheme.installEventFilter(self)
def scheme(self):
"""
Return the scheme instance on which this manager is installed.
"""
return self.__scheme
def signal_manager(self):
"""
Return the signal manager in use on the :func:`scheme`.
"""
return self.__signal_manager
def widget_for_node(self, node):
"""
Return the OWWidget instance for the scheme node.
"""
state = self.__initstate_for_node[node]
if isinstance(state, WidgetManager.Delayed):
# Create the widget now if it is still pending
state = self.__materialize(state)
return state.widget
elif isinstance(state, WidgetManager.PartiallyInitialized):
widget = state.partially_initialized_widget
log.warning(
"WidgetManager.widget_for_node: "
"Accessing a partially created widget instance. "
"This is most likely a result of explicit "
"QApplication.processEvents call from the '%s.%s' "
"widgets __init__.",
type(widget).__module__,
type(widget).__name__)
return widget
elif isinstance(state, WidgetManager.Materialized):
return state.widget
else:
assert False
def node_for_widget(self, widget):
"""
Return the SchemeNode instance for the OWWidget.
Raise a KeyError if the widget does not map to a node in the scheme.
"""
return self.__node_for_widget[widget]
def widget_properties(self, node):
"""
Return the current widget properties/settings.
Parameters
----------
node : SchemeNode
Returns
-------
settings : dict
"""
state = self.__initstate_for_node[node]
if isinstance(state, WidgetManager.Materialized):
return state.widget.settingsHandler.pack_data(state.widget)
else:
return node.properties
def set_creation_policy(self, policy):
"""
Set the widget creation policy
Parameters
----------
policy : WidgetManager.CreationPolicy
"""
if self.__creation_policy != policy:
self.__creation_policy = policy
if self.__creation_policy == WidgetManager.Immediate:
self.__init_timer.stop()
while self.__init_queue:
state = self.__init_queue.popleft()
self.__materialize(state)
elif self.__creation_policy == WidgetManager.Normal:
if not self.__init_timer.isActive() and self.__init_queue:
self.__init_timer.start()
elif self.__creation_policy == WidgetManager.OnDemand:
self.__init_timer.stop()
else:
assert False
def creation_policy(self):
"""
Return the current widget creation policy
Returns
-------
policy: WidgetManager.CreationPolicy
"""
return self.__creation_policy
def add_widget_for_node(self, node):
"""
Create a new OWWidget instance for the corresponding scheme node.
"""
state = WidgetManager.Delayed(node)
self.__initstate_for_node[node] = state
if self.__creation_policy == WidgetManager.Immediate:
self.__initstate_for_node[node] = self.__materialize(state)
elif self.__creation_policy == WidgetManager.Normal:
self.__init_queue.append(state)
if not self.__init_timer.isActive():
self.__init_timer.start()
elif self.__creation_policy == WidgetManager.OnDemand:
self.__init_queue.append(state)
def __materialize(self, state):
# Create and initialize an OWWidget for a Delayed
# widget initialization
assert isinstance(state, WidgetManager.Delayed)
if state in self.__init_queue:
self.__init_queue.remove(state)
node = state.node
widget = self.create_widget_instance(node)
self.__widgets.append(widget)
self.__widget_for_node[node] = widget
self.__node_for_widget[widget] = node
self.__initialize_widget_state(node, widget)
state = WidgetManager.Materialized(node, widget)
self.__initstate_for_node[node] = state
self.widget_for_node_added.emit(node, widget)
return state
def remove_widget_for_node(self, node):
"""
Remove the OWWidget instance for node.
"""
state = self.__initstate_for_node[node]
if isinstance(state, WidgetManager.Delayed):
del self.__initstate_for_node[node]
self.__init_queue.remove(state)
elif isinstance(state, WidgetManager.Materialized):
# Update the node's stored settings/properties dict before
# removing the widget.
# TODO: Update/sync whenever the widget settings change.
node.properties = self._widget_settings(state.widget)
self.__widgets.remove(state.widget)
del self.__initstate_for_node[node]
del self.__widget_for_node[node]
del self.__node_for_widget[state.widget]
node.title_changed.disconnect(state.widget.setCaption)
state.widget.progressBarValueChanged.disconnect(node.set_progress)
del state.widget._Report__report_view
self.widget_for_node_removed.emit(node, state.widget)
self._delete_widget(state.widget)
elif isinstance(state, WidgetManager.PartiallyInitialized):
widget = state.partially_initialized_widget
raise RuntimeError(
"A widget/node {} was removed while being initialized. "
"This is most likely a result of an explicit "
"QApplication.processEvents call from the '{}.{}' "
"widgets __init__.\n".format(state.node.title,
type(widget).__module__,
type(widget).__init__))
def _widget_settings(self, widget):
return widget.settingsHandler.pack_data(widget)
def _delete_widget(self, widget):
"""
Delete the OWBaseWidget instance.
"""
widget.close()
# Save settings to user global settings.
widget.saveSettings()
# Notify the widget it will be deleted.
widget.onDeleteWidget()
state = self.__widget_processing_state[widget]
if state & WidgetManager._DelayDeleteMask:
# If the widget is in an update loop and/or blocking we
# delay the scheduled deletion until the widget is done.
log.debug(
"Widget %s removed but still in state :%s. "
"Deferring deletion.", widget, state)
self.__delay_delete.add(widget)
else:
widget.deleteLater()
del self.__widget_processing_state[widget]
def create_widget_instance(self, node):
"""
Create a OWWidget instance for the node.
"""
desc = node.description
klass = widget = None
initialized = False
error = None
# First try to actually retrieve the class.
try:
klass = name_lookup(desc.qualified_name)
except (ImportError, AttributeError):
sys.excepthook(*sys.exc_info())
error = "Could not import {0!r}\n\n{1}".format(
node.description.qualified_name, traceback.format_exc())
except Exception:
sys.excepthook(*sys.exc_info())
error = "An unexpected error during import of {0!r}\n\n{1}".format(
node.description.qualified_name, traceback.format_exc())
if klass is None:
widget = mock_error_owwidget(node, error)
initialized = True
if widget is None:
log.info("WidgetManager: Creating '%s.%s' instance '%s'.",
klass.__module__, klass.__name__, node.title)
widget = klass.__new__(
klass,
None,
captionTitle=node.title,
signal_manager=self.signal_manager(),
stored_settings=node.properties,
# NOTE: env is a view of the real env and reflects
# changes to the environment.
env=self.scheme().runtime_env())
initialized = False
# Init the node/widget mapping and state before calling __init__
# Some OWWidgets might already send data in the constructor
# (should this be forbidden? Raise a warning?) triggering the signal
# manager which would request the widget => node mapping or state
# Furthermore they can (though they REALLY REALLY REALLY should not)
# explicitly call qApp.processEvents.
assert node not in self.__widget_for_node
self.__widget_for_node[node] = widget
self.__node_for_widget[widget] = node
self.__widget_processing_state[widget] = WidgetManager.Initializing
self.__initstate_for_node[node] = \
WidgetManager.PartiallyInitialized(node, widget)
if not initialized:
try:
widget.__init__()
except Exception:
sys.excepthook(*sys.exc_info())
msg = traceback.format_exc()
msg = "Could not create {0!r}\n\n{1}".format(
node.description.name, msg)
# remove state tracking for widget ...
del self.__widget_for_node[node]
del self.__node_for_widget[widget]
del self.__widget_processing_state[widget]
# ... and substitute it with a mock error widget.
widget = mock_error_owwidget(node, msg)
self.__widget_for_node[node] = widget
self.__node_for_widget[widget] = node
self.__widget_processing_state[widget] = 0
self.__initstate_for_node[node] = \
WidgetManager.Materialized(node, widget)
self.__initstate_for_node[node] = \
WidgetManager.Materialized(node, widget)
# Clear Initializing flag
self.__widget_processing_state[widget] &= ~WidgetManager.Initializing
node.title_changed.connect(widget.setCaption)
# Widget's info/warning/error messages.
widget.messageActivated.connect(self.__on_widget_state_changed)
widget.messageDeactivated.connect(self.__on_widget_state_changed)
# Widget's statusTip
node.set_status_message(widget.statusMessage())
widget.statusMessageChanged.connect(node.set_status_message)
# Widget's progress bar value state.
widget.progressBarValueChanged.connect(node.set_progress)
# Widget processing state (progressBarInit/Finished)
# and the blocking state.
widget.processingStateChanged.connect(
self.__on_processing_state_changed)
widget.blockingStateChanged.connect(self.__on_blocking_state_changed)
if widget.isBlocking():
# A widget can already enter blocking state in __init__
self.__widget_processing_state[widget] |= self.BlockingUpdate
if widget.processingState != 0:
# It can also start processing (initialization of resources, ...)
self.__widget_processing_state[widget] |= self.ProcessingUpdate
node.set_processing_state(1)
node.set_progress(widget.progressBarValue)
# Install a help shortcut on the widget
help_action = widget.findChild(QAction, "action-help")
if help_action is not None:
help_action.setEnabled(True)
help_action.setVisible(True)
help_action.triggered.connect(self.__on_help_request)
# Up shortcut (activate/open parent)
up_shortcut = QShortcut(QKeySequence(Qt.ControlModifier + Qt.Key_Up),
widget)
up_shortcut.activated.connect(self.__on_activate_parent)
# Call setters only after initialization.
widget.setWindowIcon(icon_loader.from_description(desc).get(desc.icon))
widget.setCaption(node.title)
# befriend class Report
widget._Report__report_view = self.scheme().report_view
self.__set_float_on_top_flag(widget)
# Schedule an update with the signal manager, due to the cleared
# implicit Initializing flag
self.signal_manager()._update()
return widget
def node_processing_state(self, node):
"""
Return the processing state flags for the node.
Same as `manager.widget_processing_state(manger.widget_for_node(node))`
"""
state = self.__initstate_for_node[node]
if isinstance(state, WidgetManager.Materialized):
return self.__widget_processing_state[state.widget]
elif isinstance(state, WidgetManager.PartiallyInitialized):
return self.__widget_processing_state[
state.partially_initialized_widget]
else:
return WidgetManager.Initializing
def widget_processing_state(self, widget):
"""
Return the processing state flags for the widget.
The state is an bitwise or of `InputUpdate` and `BlockingUpdate`.
"""
return self.__widget_processing_state[widget]
def set_float_widgets_on_top(self, float_on_top):
"""
Set `Float Widgets on Top` flag on all widgets.
"""
self.__float_widgets_on_top = float_on_top
for widget in self.__widget_for_node.values():
self.__set_float_on_top_flag(widget)
def __create_delayed(self):
if self.__init_queue:
state = self.__init_queue.popleft()
node = state.node
self.__initstate_for_node[node] = self.__materialize(state)
if self.__creation_policy == WidgetManager.Normal and \
self.__init_queue:
# restart the timer if pending widgets still in the queue
self.__init_timer.start()
def eventFilter(self, receiver, event):
if event.type() == QEvent.Close and receiver is self.__scheme:
self.signal_manager().stop()
# Notify the widget instances.
for widget in list(self.__widget_for_node.values()):
widget.close()
widget.saveSettings()
widget.onDeleteWidget()
widget.deleteLater()
return QObject.eventFilter(self, receiver, event)
def __on_help_request(self):
"""
Help shortcut was pressed. We send a `QWhatsThisClickedEvent` to
the scheme and hope someone responds to it.
"""
# Sender is the QShortcut, and parent the OWBaseWidget
widget = self.sender().parent()
try:
node = self.node_for_widget(widget)
except KeyError:
pass
else:
qualified_name = node.description.qualified_name
help_url = "help://search?" + urlencode({"id": qualified_name})
event = QWhatsThisClickedEvent(help_url)
QCoreApplication.sendEvent(self.scheme(), event)
def __on_activate_parent(self):
"""
Activate parent shortcut was pressed.
"""
event = ActivateParentEvent()
QCoreApplication.sendEvent(self.scheme(), event)
def __initialize_widget_state(self, node, widget):
"""
Initialize the tracked info/warning/error message state.
"""
for message_group in widget.message_groups:
message = user_message_from_state(message_group)
if message:
node.set_state_message(message)
def __on_widget_state_changed(self, msg):
"""
The OWBaseWidget info/warning/error state has changed.
"""
widget = msg.group.widget
try:
node = self.node_for_widget(widget)
except KeyError:
pass
else:
self.__initialize_widget_state(node, widget)
def __on_processing_state_changed(self, state):
"""
A widget processing state has changed (progressBarInit/Finished)
"""
widget = self.sender()
if state:
self.__widget_processing_state[widget] |= self.ProcessingUpdate
else:
self.__widget_processing_state[widget] &= ~self.ProcessingUpdate
# propagate the change to the workflow model.
try:
# we can still track widget state after it was removed from the
# workflow model (`__delay_delete`)
node = self.node_for_widget(widget)
except KeyError:
pass
else:
self.__update_node_processing_state(node)
def __on_processing_started(self, node):
"""
Signal manager entered the input update loop for the node.
"""
widget = self.widget_for_node(node)
# Remember the widget instance. The node and the node->widget mapping
# can be removed between this and __on_processing_finished.
self.__updating_widget = widget
self.__widget_processing_state[widget] |= self.InputUpdate
self.__update_node_processing_state(node)
def __on_processing_finished(self, node):
"""
Signal manager exited the input update loop for the node.
"""
widget = self.__updating_widget
self.__widget_processing_state[widget] &= ~self.InputUpdate
if widget in self.__node_for_widget:
self.__update_node_processing_state(node)
elif widget in self.__delay_delete:
self.__try_delete(widget)
else:
raise ValueError("%r is not managed" % widget)
self.__updating_widget = None
def __on_blocking_state_changed(self, state):
"""
OWWidget blocking state has changed.
"""
if not state:
# schedule an update pass.
self.signal_manager()._update()
widget = self.sender()
if state:
self.__widget_processing_state[widget] |= self.BlockingUpdate
else:
self.__widget_processing_state[widget] &= ~self.BlockingUpdate
if widget in self.__node_for_widget:
node = self.node_for_widget(widget)
self.__update_node_processing_state(node)
elif widget in self.__delay_delete:
self.__try_delete(widget)
def __update_node_processing_state(self, node):
"""
Update the `node.processing_state` to reflect the widget state.
"""
state = self.node_processing_state(node)
node.set_processing_state(1 if state else 0)
def __try_delete(self, widget):
if not (self.__widget_processing_state[widget]
& WidgetManager._DelayDeleteMask):
log.debug("Delayed delete for widget %s", widget)
self.__delay_delete.remove(widget)
del self.__widget_processing_state[widget]
widget.blockingStateChanged.disconnect(
self.__on_blocking_state_changed)
widget.processingStateChanged.disconnect(
self.__on_processing_state_changed)
widget.deleteLater()
def __on_env_changed(self, key, newvalue, oldvalue):
# Notify widgets of a runtime environment change
for widget in self.__widget_for_node.values():
widget.workflowEnvChanged(key, newvalue, oldvalue)
def __set_float_on_top_flag(self, widget):
"""Set or unset widget's float on top flag"""
should_float_on_top = self.__float_widgets_on_top
if hasattr(qApp, "applicationState"):
# only float on top when the application is active
# available in Qt >= 5.2
should_float_on_top &= qApp.applicationState(
) == Qt.ApplicationActive
float_on_top = widget.windowFlags() & Qt.WindowStaysOnTopHint
if float_on_top == should_float_on_top:
return
widget_was_visible = widget.isVisible()
if should_float_on_top:
widget.setWindowFlags(Qt.WindowStaysOnTopHint)
else:
widget.setWindowFlags(widget.windowFlags()
& ~Qt.WindowStaysOnTopHint)
# Changing window flags hid the widget
if widget_was_visible:
widget.show()
class OWtSNE(OWDataProjectionWidget):
name = "t-SNE"
description = "Two-dimensional data projection with t-SNE."
icon = "icons/TSNE.svg"
priority = 920
keywords = ["tsne"]
settings_version = 3
max_iter = Setting(300)
perplexity = Setting(30)
pca_components = Setting(20)
GRAPH_CLASS = OWtSNEGraph
graph = SettingProvider(OWtSNEGraph)
embedding_variables_names = ("t-SNE-x", "t-SNE-y")
#: Runtime state
Running, Finished, Waiting = 1, 2, 3
class Outputs(OWDataProjectionWidget.Outputs):
preprocessor = Output("Preprocessor", Preprocess)
class Error(OWDataProjectionWidget.Error):
not_enough_rows = Msg("Input data needs at least 2 rows")
constant_data = Msg("Input data is constant")
no_attributes = Msg("Data has no attributes")
out_of_memory = Msg("Out of memory")
optimization_error = Msg("Error during optimization\n{}")
no_valid_data = Msg("No projection due to no valid data")
def __init__(self):
super().__init__()
self.pca_data = None
self.projection = None
self.__invalidated = True
self.__update_loop = None
# timer for scheduling updates
self.__timer = QTimer(self, singleShot=True, interval=1,
timeout=self.__next_step)
self.__state = OWtSNE.Waiting
self.__in_next_step = False
self.__draw_similar_pairs = False
def _add_controls(self):
self._add_controls_start_box()
super()._add_controls()
# Because sc data frequently has many genes,
# showing all attributes in combo boxes can cause problems
# QUICKFIX: Remove a separator and attributes from order
# (leaving just the class and metas)
self.models = self.graph.gui.points_models
for model in self.models:
model.order = model.order[:-2]
def _add_controls_start_box(self):
box = gui.vBox(self.controlArea, True)
form = QFormLayout(
labelAlignment=Qt.AlignLeft,
formAlignment=Qt.AlignLeft,
fieldGrowthPolicy=QFormLayout.AllNonFixedFieldsGrow,
verticalSpacing=10
)
form.addRow(
"Max iterations:",
gui.spin(box, self, "max_iter", 1, 2000, step=50))
form.addRow(
"Perplexity:",
gui.spin(box, self, "perplexity", 1, 100, step=1))
box.layout().addLayout(form)
gui.separator(box, 10)
self.runbutton = gui.button(box, self, "Run", callback=self._toggle_run)
gui.separator(box, 10)
gui.hSlider(box, self, "pca_components", label="PCA components:",
minValue=2, maxValue=50, step=1)
def set_data(self, data):
self.__invalidated = not (self.data and data and
np.array_equal(self.data.X, data.X))
super().set_data(data)
def check_data(self):
def error(err):
err()
self.data = None
super().check_data()
if self.data is not None:
if len(self.data) < 2:
error(self.Error.not_enough_rows)
elif not self.data.domain.attributes:
error(self.Error.no_attributes)
elif not self.data.is_sparse() and \
np.allclose(self.data.X - self.data.X[0], 0):
error(self.Error.constant_data)
elif not self.data.is_sparse() and \
np.all(~np.isfinite(self.data.X)):
error(self.Error.no_valid_data)
def get_embedding(self):
if self.data is None:
self.valid_data = None
return None
elif self.projection is None:
embedding = np.random.normal(size=(len(self.data), 2))
else:
embedding = self.projection.embedding.X
self.valid_data = np.ones(len(embedding), dtype=bool)
return embedding
def _toggle_run(self):
if self.__state == OWtSNE.Running:
self.stop()
self.commit()
else:
self.start()
def start(self):
if not self.data or self.__state == OWtSNE.Running:
self.graph.update_coordinates()
elif self.__state in (OWtSNE.Finished, OWtSNE.Waiting):
self.__start()
def stop(self):
if self.__state == OWtSNE.Running:
self.__set_update_loop(None)
def pca_preprocessing(self):
if self.pca_data is not None and \
self.pca_data.X.shape[1] == self.pca_components:
return
pca = PCA(n_components=self.pca_components, random_state=0)
model = pca(self.data)
self.pca_data = model(self.data)
def __start(self):
self.pca_preprocessing()
initial = 'random' if self.projection is None \
else self.projection.embedding.X
step_size = 50
def update_loop(data, max_iter, step, embedding):
# NOTE: this code MUST NOT call into QApplication.processEvents
done = False
iterations_done = 0
while not done:
step_iter = min(max_iter - iterations_done, step)
projection = compute_tsne(
data, self.perplexity, step_iter, embedding)
embedding = projection.embedding.X
iterations_done += step_iter
if iterations_done >= max_iter:
done = True
yield projection, iterations_done / max_iter
self.__set_update_loop(update_loop(
self.pca_data, self.max_iter, step_size, initial))
self.progressBarInit(processEvents=None)
def __set_update_loop(self, loop):
if self.__update_loop is not None:
self.__update_loop.close()
self.__update_loop = None
self.progressBarFinished(processEvents=None)
self.__update_loop = loop
if loop is not None:
self.setBlocking(True)
self.progressBarInit(processEvents=None)
self.setStatusMessage("Running")
self.runbutton.setText("Stop")
self.__state = OWtSNE.Running
self.__timer.start()
else:
self.setBlocking(False)
self.setStatusMessage("")
self.runbutton.setText("Start")
self.__state = OWtSNE.Finished
self.__timer.stop()
def __next_step(self):
if self.__update_loop is None:
return
assert not self.__in_next_step
self.__in_next_step = True
loop = self.__update_loop
self.Error.out_of_memory.clear()
self.Error.optimization_error.clear()
try:
projection, progress = next(self.__update_loop)
assert self.__update_loop is loop
except StopIteration:
self.__set_update_loop(None)
self.unconditional_commit()
except MemoryError:
self.Error.out_of_memory()
self.__set_update_loop(None)
except Exception as exc:
self.Error.optimization_error(str(exc))
self.__set_update_loop(None)
else:
self.progressBarSet(100.0 * progress, processEvents=None)
self.projection = projection
self.graph.update_coordinates()
self.graph.update_density()
# schedule next update
self.__timer.start()
self.__in_next_step = False
def handleNewSignals(self):
if self.__invalidated:
self.__invalidated = False
self.setup_plot()
self.start()
else:
self.graph.update_point_props()
self.commit()
def commit(self):
super().commit()
self.send_preprocessor()
def _get_projection_data(self):
if self.data is None or self.projection is None:
return None
data = self.data.transform(
Domain(self.data.domain.attributes,
self.data.domain.class_vars,
self.data.domain.metas + self.projection.domain.attributes))
data.metas[:, -2:] = self.get_embedding()
return data
def send_preprocessor(self):
prep = None
if self.data is not None and self.projection is not None:
prep = ApplyDomain(self.projection.domain, self.projection.name)
self.Outputs.preprocessor.send(prep)
def clear(self):
if self.__invalidated:
super().clear()
self.__set_update_loop(None)
self.__state = OWtSNE.Waiting
self.pca_data = None
self.projection = None
@classmethod
def migrate_settings(cls, settings, version):
if version < 3:
if "selection_indices" in settings:
settings["selection"] = settings["selection_indices"]
@classmethod
def migrate_context(cls, context, version):
if version < 3:
values = context.values
values["attr_color"] = values["graph"]["attr_color"]
values["attr_size"] = values["graph"]["attr_size"]
values["attr_shape"] = values["graph"]["attr_shape"]
values["attr_label"] = values["graph"]["attr_label"]
class SignalManager(QObject):
"""
SignalManager handles the runtime signal propagation for a :class:`.Scheme`
instance.
Note
----
If a scheme instance is passed as a parent to the constructor it is also
set as the workflow model.
"""
class State(enum.IntEnum):
"""
SignalManager state flags.
.. seealso:: :func:`SignalManager.state()`
"""
#: The manager is running, i.e. it propagates signals
Running = 0
#: The manager is stopped. It does not track node output changes,
#: and does not deliver signals to dependent nodes
Stopped = 1
#: The manager is paused. It still tracks node output changes, but
#: does not deliver new signals to dependent nodes. The pending signals
#: will be delivered once it enters Running state again
Paused = 2
#: The manager is running, i.e. it propagates signals
Running = State.Running
#: The manager is stopped. It does not track node ouput changes,
#: and does not deliver signals to dependent nodes
Stopped = State.Stopped
#: The manager is paused. It still tracks node output changes, but
#: does not deliver new signals to dependent nodes. The pending signals
#: will be delivered once it enters Running state again
Paused = Stopped.Paused
# unused; back-compatibility
Error = 3
class RuntimeState(enum.IntEnum):
"""
SignalManager runtime state.
See Also
--------
SignalManager.runtime_state
"""
#: Waiting, idle state. The signal queue is empty
Waiting = 0
#: ...
Processing = 1
Waiting = RuntimeState.Waiting
Processing = RuntimeState.Processing
#: Emitted when the state of the signal manager changes.
stateChanged = pyqtSignal(int)
#: Emitted when signals are added to the queue.
updatesPending = pyqtSignal()
#: Emitted right before a `SchemeNode` instance has its inputs updated.
processingStarted = pyqtSignal([], [SchemeNode])
#: Emitted right after a `SchemeNode` instance has had its inputs updated.
processingFinished = pyqtSignal([], [SchemeNode])
#: Emitted when `SignalManager`'s runtime state changes.
runtimeStateChanged = pyqtSignal(int)
def __init__(self, parent=None, **kwargs):
super().__init__(parent, **kwargs)
self.__workflow = None # type: Optional[Scheme]
self.__input_queue = [] # type: List[Signal]
# mapping a node to its current outputs
self.__node_outputs = {} # type: Dict[Node, Dict[OutputSignal, Dict[Any, Any]]]
self.__state = SignalManager.Running
self.__runtime_state = SignalManager.Waiting
# A flag indicating if UpdateRequest event should be rescheduled
self.__reschedule = False
self.__update_timer = QTimer(self, interval=100, singleShot=True)
self.__update_timer.timeout.connect(self.__process_next)
if isinstance(parent, Scheme):
self.set_workflow(parent)
def _can_process(self):
"""
Return a bool indicating if the manger can enter the main
processing loop.
"""
return self.__state not in [SignalManager.Error, SignalManager.Stopped]
def workflow(self):
# type: () -> Scheme
"""
Return the :class:`Scheme` instance.
"""
return self.__workflow
#: Alias
scheme = workflow
def set_workflow(self, workflow):
# type: (Scheme) -> None
"""
Set the workflow model.
Parameters
----------
workflow : Scheme
"""
if workflow is self.__workflow:
return
if self.__workflow is not None:
for link in self.__workflow.links:
link.enabled_changed.disconnect(self.link_enabled_changed)
self.__workflow.node_added.disconnect(self.on_node_added)
self.__workflow.node_removed.disconnect(self.on_node_removed)
self.__workflow.link_added.disconnect(self.link_added)
self.__workflow.link_removed.disconnect(self.link_removed)
self.__workflow.removeEventFilter(self)
self.__node_outputs = {}
self.__input_queue = []
self.__workflow = workflow
if workflow is not None:
workflow.node_added.connect(self.on_node_added)
workflow.node_removed.connect(self.on_node_removed)
workflow.link_added.connect(self.link_added)
workflow.link_removed.connect(self.link_removed)
for node in workflow.nodes:
self.__node_outputs[node] = defaultdict(dict)
for link in workflow.links:
link.enabled_changed.connect(self.link_enabled_changed)
workflow.installEventFilter(self)
def has_pending(self):
"""
Does the manager have any signals to deliver?
"""
return bool(self.__input_queue)
def start(self):
"""
Start the update loop.
Note
----
The updates will not happen until the control reaches the Qt event
loop.
"""
if self.__state != SignalManager.Running:
self.__state = SignalManager.Running
self.stateChanged.emit(SignalManager.Running)
self._update()
def stop(self):
"""
Stop the update loop.
Note
----
If the `SignalManager` is currently in `process_queues` it will
still update all current pending signals, but will not re-enter
until `start()` is called again.
"""
if self.__state != SignalManager.Stopped:
self.__state = SignalManager.Stopped
self.stateChanged.emit(SignalManager.Stopped)
self.__update_timer.stop()
def pause(self):
"""
Pause the delivery of signals.
"""
if self.__state != SignalManager.Paused:
self.__state = SignalManager.Paused
self.stateChanged.emit(SignalManager.Paused)
self.__update_timer.stop()
def resume(self):
"""
Resume the delivery of signals.
"""
if self.__state == SignalManager.Paused:
self.__state = SignalManager.Running
self.stateChanged.emit(self.__state)
self._update()
def step(self):
"""
Deliver signals to a single node (only applicable while the `state()`
is `Paused`).
"""
if self.__state == SignalManager.Paused:
self.process_queued()
def state(self):
# type: () -> State
"""
Return the current state.
Return
------
state : SignalManager.State
"""
return self.__state
def _set_runtime_state(self, state):
"""
Set the runtime state.
Should only be called by `SignalManager` implementations.
"""
if self.__runtime_state != state:
self.__runtime_state = state
self.runtimeStateChanged.emit(self.__runtime_state)
def runtime_state(self):
# type: () -> RuntimeState
"""
Return the runtime state. This can be `SignalManager.Waiting`
or `SignalManager.Processing`.
"""
return self.__runtime_state
def on_node_removed(self, node):
# remove all pending input signals for node so we don't get
# stale references in process_node.
# NOTE: This does not remove output signals for this node. In
# particular the final 'None' will be delivered to the sink
# nodes even after the source node is no longer in the scheme.
log.info("Removing pending signals for '%s'.", node.title)
self.remove_pending_signals(node)
del self.__node_outputs[node]
def on_node_added(self, node):
self.__node_outputs[node] = defaultdict(dict)
def link_added(self, link):
# push all current source values to the sink
link.set_runtime_state(SchemeLink.Empty)
if link.enabled:
log.info("Scheduling signal data update for '%s'.", link)
self._schedule(self.signals_on_link(link))
self._update()
link.enabled_changed.connect(self.link_enabled_changed)
def link_removed(self, link):
# purge all values in sink's queue
log.info("Scheduling signal data purge (%s).", link)
self.purge_link(link)
link.enabled_changed.disconnect(self.link_enabled_changed)
def link_enabled_changed(self, enabled):
if enabled:
link = self.sender()
log.info("Link %s enabled. Scheduling signal data update.", link)
self._schedule(self.signals_on_link(link))
def signals_on_link(self, link):
# type: (SchemeLink) -> List[Signal]
"""
Return :class:`Signal` instances representing the current values
present on the link.
"""
items = self.link_contents(link)
signals = []
for key, value in items.items():
signals.append(Signal(link, value, key))
return signals
def link_contents(self, link):
"""
Return the contents on link.
"""
node, channel = link.source_node, link.source_channel
if node in self.__node_outputs:
return self.__node_outputs[node][channel]
else:
# if the the node was already removed its tracked outputs in
# __node_outputs are cleared, however the final 'None' signal
# deliveries for the link are left in the _input_queue.
pending = [sig for sig in self.__input_queue
if sig.link is link]
return {sig.id: sig.value for sig in pending}
def send(self, node, channel, value, id):
# type: (SchemeNode, OutputSignal, Any, Any) -> None
"""
Send the `value` with `id` on an output `channel` from node.
Schedule the signal delivery to all dependent nodes
Parameters
----------
node : SchemeNode
The originating node.
channel : OutputSignal
The nodes output on which the value is sent.
value : Any
The value to send,
id : Any
Signal id.
"""
log.debug("%r sending %r (id: %r) on channel %r",
node.title, type(value), id, channel.name)
scheme = self.scheme()
self.__node_outputs[node][channel][id] = value
links = scheme.find_links(source_node=node, source_channel=channel)
links = filter(is_enabled, links)
signals = []
for link in links:
signals.append(Signal(link, value, id))
self._schedule(signals)
def purge_link(self, link):
"""
Purge the link (send None for all ids currently present)
"""
contents = self.link_contents(link)
ids = contents.keys()
signals = [Signal(link, None, id) for id in ids]
self._schedule(signals)
def _schedule(self, signals):
"""
Schedule a list of :class:`Signal` for delivery.
"""
self.__input_queue.extend(signals)
for link in {sig.link for sig in signals}:
# update the SchemeLink's runtime state flags
contents = self.link_contents(link)
if any(value is not None for value in contents.values()):
state = SchemeLink.Active
else:
state = SchemeLink.Empty
link.set_runtime_state(state | SchemeLink.Pending)
if signals:
self.updatesPending.emit()
self._update()
def _update_link(self, link):
"""
Schedule update of a single link.
"""
signals = self.signals_on_link(link)
self._schedule(signals)
def process_queued(self, max_nodes=None):
"""
Process queued signals.
Take one node node from the pending input queue and deliver
all scheduled signals.
"""
if not (max_nodes is None or max_nodes == 1):
warnings.warn(
"`max_nodes` is deprecated and unused (will always equal 1)",
DeprecationWarning, stacklevel=2)
if self.__runtime_state == SignalManager.Processing:
raise RuntimeError("Cannot re-enter 'process_queued'")
if not self._can_process():
raise RuntimeError("Can't process in state %i" % self.__state)
log.info("SignalManager: Processing queued signals")
node_update_front = self.node_update_front()
log.debug("SignalManager: Nodes eligible for update %s",
[node.title for node in node_update_front])
if node_update_front:
node = node_update_front[0]
self._set_runtime_state(SignalManager.Processing)
try:
self.process_node(node)
finally:
self._set_runtime_state(SignalManager.Waiting)
def process_node(self, node):
"""
Process pending input signals for `node`.
"""
signals_in = self.pending_input_signals(node)
self.remove_pending_signals(node)
signals_in = self.compress_signals(signals_in)
log.debug("Processing %r, sending %i signals.",
node.title, len(signals_in))
# Clear the link's pending flag.
for link in {sig.link for sig in signals_in}:
link.set_runtime_state(link.runtime_state() & ~SchemeLink.Pending)
def process_dynamic(signals):
# type: (List[Signal]) -> List[Signal]
"""
Process dynamic signals; Update the link's dynamic_enabled flag if
the value is valid; replace values that do not type check with
`None`
"""
res = []
for sig in signals:
# Check and update the dynamic link state
link = sig.link
if sig.link.is_dynamic():
link.dynamic_enabled = can_enable_dynamic(link, sig.value)
if not link.dynamic_enabled:
# Send None instead
sig = Signal(link, None, sig.id)
res.append(sig)
return res
signals_in = process_dynamic(signals_in)
assert ({sig.link for sig in self.__input_queue}
.intersection({sig.link for sig in signals_in}) == set([]))
self.processingStarted.emit()
self.processingStarted[SchemeNode].emit(node)
try:
self.send_to_node(node, signals_in)
finally:
self.processingFinished.emit()
self.processingFinished[SchemeNode].emit(node)
def compress_signals(self, signals):
# type: (List[Signal]) -> List[Signal]
"""
Compress a list of :class:`Signal` instances to be delivered.
Before the signal values are delivered to the sink node they can be
optionally `compressed`, i.e. values can be merged or dropped
depending on the execution semantics.
The input list is in the order that the signals were enqueued.
The base implementation returns the list unmodified.
Parameters
----------
signals : List[Signal]
Return
------
signals : List[Signal]
"""
return signals
def send_to_node(self, node, signals):
# type: (SchemeNode, List[Signal]) -> None
"""
Abstract. Reimplement in subclass.
Send/notify the `node` instance (or whatever object/instance it is a
representation of) that it has new inputs as represented by the
`signals` list).
Parameters
----------
node : SchemeNode
signals : List[Signal]
"""
raise NotImplementedError
def is_pending(self, node):
# type: (SchemeNode) -> bool
"""
Is `node` (class:`SchemeNode`) scheduled for processing (i.e.
it has incoming pending signals).
Parameters
----------
node : SchemeNode
Returns
-------
pending : bool
"""
return node in [signal.link.sink_node for signal in self.__input_queue]
def pending_nodes(self):
# type: () -> List[SchemeNode]
"""
Return a list of pending nodes.
The nodes are returned in the order they were enqueued for
signal delivery.
Returns
-------
nodes : List[SchemeNode]
"""
return list(unique(sig.link.sink_node for sig in self.__input_queue))
def pending_input_signals(self, node):
# type: (SchemeNode) -> List[Signal]
"""
Return a list of pending input signals for node.
"""
return [signal for signal in self.__input_queue
if node is signal.link.sink_node]
def remove_pending_signals(self, node):
# type: (SchemeNode) -> None
"""
Remove pending signals for `node`.
"""
for signal in self.pending_input_signals(node):
try:
self.__input_queue.remove(signal)
except ValueError:
pass
def blocking_nodes(self):
# type: () -> List[SchemeNode]
"""
Return a list of nodes in a blocking state.
"""
scheme = self.scheme()
return [node for node in scheme.nodes if self.is_blocking(node)]
def is_blocking(self, node):
# type: (SchemeNode) -> bool
"""
Is the node in `blocking` state.
Is it currently in a state where will produce new outputs and
therefore no signals should be delivered to dependent nodes until
it does so.
The default implementation returns False.
"""
# TODO: this needs a different name
return False
def node_update_front(self):
# type: () -> List[SchemeNode]
"""
Return a list of nodes on the update front, i.e. nodes scheduled for
an update that have no ancestor which is either itself scheduled
for update or is in a blocking state).
Note
----
The node's ancestors are only computed over enabled links.
"""
scheme = self.scheme()
def expand(node):
return [link.sink_node
for link in scheme.find_links(source_node=node)
if link.enabled]
components = strongly_connected_components(scheme.nodes, expand)
node_scc = {node: scc for scc in components for node in scc}
def isincycle(node):
return len(node_scc[node]) > 1
# a list of all nodes currently active/executing a task.
blocking_nodes = set(self.blocking_nodes())
dependents = partial(dependent_nodes, scheme)
blocked_nodes = reduce(set.union,
map(dependents, blocking_nodes),
set(blocking_nodes))
pending = self.pending_nodes()
pending_downstream = set()
for n in pending:
depend = set(dependents(n))
if isincycle(n):
# a pending node in a cycle would would have a circular
# dependency on itself, preventing any progress being made
# by the workflow execution.
cc = node_scc[n]
depend -= set(cc)
pending_downstream.update(depend)
log.debug("Pending nodes: %s", pending)
log.debug("Blocking nodes: %s", blocking_nodes)
noneligible = pending_downstream | blocked_nodes
return [node for node in pending if node not in noneligible]
@Slot()
def __process_next(self):
if not self.__state == SignalManager.Running:
log.debug("Received 'UpdateRequest' while not in 'Running' state")
return
if self.__runtime_state == SignalManager.Processing:
# This happens if QCoreApplication.processEvents is called from
# the input handlers. A `__process_next` must be rescheduled when
# exiting process_queued.
log.warning("Received 'UpdateRequest' while in 'process_queued'. "
"An update will be re-scheduled when exiting the "
"current update.")
self.__reschedule = True
return
nbusy = len(self.blocking_nodes())
log.info("'UpdateRequest' event, queued signals: %i, nbusy: %i "
"(MAX_CONCURRENT: %i)",
len(self.__input_queue), nbusy, MAX_CONCURRENT)
if self.__input_queue and nbusy < MAX_CONCURRENT:
self.process_queued()
if self.__reschedule and self.__state == SignalManager.Running:
self.__reschedule = False
log.debug("Rescheduling signal update")
self.__update_timer.start()
nbusy = len(self.blocking_nodes())
if self.node_update_front() and nbusy < MAX_CONCURRENT:
log.debug("More nodes are eligible for an update. "
"Scheduling another update.")
self._update()
def _update(self):
"""
Schedule processing at a later time.
"""
if self.__state == SignalManager.Running and \
not self.__update_timer.isActive():
self.__update_timer.start()
def eventFilter(self, receiver, event):
"""
Reimplemented.
"""
if event.type() == QEvent.DeferredDelete \
and receiver is self.__workflow:
# ?? This is really, probably, mostly, likely not needed. Should
# just raise error from __process_next.
state = self.runtime_state()
if state == SignalManager.Processing:
log.critical(
"The workflow model %r received a deferred delete request "
"while performing an input update. "
"Deferring a 'DeferredDelete' event for the workflow "
"until SignalManager exits the current update step.",
self.__workflow
)
warnings.warn(
"The workflow model received a deferred delete request "
"while updating inputs. In the future this will raise "
"a RuntimeError", _FutureRuntimeWarning,
)
event.setAccepted(False)
self.processingFinished.connect(self.__workflow.deleteLater)
self.stop()
return True
return super().eventFilter(receiver, event)
class SignalManager(QObject):
"""
Handle all runtime signal propagation for a :clas:`Scheme` instance.
The scheme must be passed to the constructor and will become the parent
of this object. Furthermore this should happen before any items
(nodes, links) are added to the scheme.
"""
Running, Stoped, Paused, Error = range(4)
"""SignalManger state flags."""
Waiting, Processing = range(2)
"""SignalManager runtime state flags."""
stateChanged = Signal(int)
"""Emitted when the state of the signal manager changes."""
updatesPending = Signal()
"""Emitted when signals are added to the queue."""
processingStarted = Signal([], [SchemeNode])
"""Emitted right before a `SchemeNode` instance has its inputs
updated.
"""
processingFinished = Signal([], [SchemeNode])
"""Emitted right after a `SchemeNode` instance has had its inputs
updated.
"""
runtimeStateChanged = Signal(int)
"""Emitted when `SignalManager`'s runtime state changes."""
def __init__(self, scheme):
assert(scheme)
QObject.__init__(self, scheme)
self._input_queue = []
# mapping a node to it's current outputs
# {node: {channel: {id: signal_value}}}
self._node_outputs = {}
self.__state = SignalManager.Running
self.__runtime_state = SignalManager.Waiting
# A flag indicating if UpdateRequest event should be rescheduled
self.__reschedule = False
self.__update_timer = QTimer(self, interval=100, singleShot=True)
self.__update_timer.timeout.connect(self.__process_next)
def _can_process(self):
"""
Return a bool indicating if the manger can enter the main
processing loop.
"""
return self.__state not in [SignalManager.Error, SignalManager.Stoped]
def scheme(self):
"""
Return the parent class:`Scheme` instance.
"""
return self.parent()
def start(self):
"""
Start the update loop.
.. note:: The updates will not happen until the control reaches
the Qt event loop.
"""
if self.__state != SignalManager.Running:
self.__state = SignalManager.Running
self.stateChanged.emit(SignalManager.Running)
self._update()
def stop(self):
"""
Stop the update loop.
.. note:: If the `SignalManager` is currently in `process_queues` it
will still update all current pending signals, but will not
re-enter until `start()` is called again
"""
if self.__state != SignalManager.Stoped:
self.__state = SignalManager.Stoped
self.stateChanged.emit(SignalManager.Stoped)
self.__update_timer.stop()
def pause(self):
"""
Pause the updates.
"""
if self.__state != SignalManager.Paused:
self.__state = SignalManager.Paused
self.stateChanged.emit(SignalManager.Paused)
self.__update_timer.stop()
def resume(self):
if self.__state == SignalManager.Paused:
self.__state = SignalManager.Running
self.stateChanged.emit(self.__state)
self._update()
def step(self):
if self.__state == SignalManager.Paused:
self.process_queued()
def state(self):
"""
Return the current state.
"""
return self.__state
def _set_runtime_state(self, state):
"""
Set the runtime state.
Should only be called by `SignalManager` implementations.
"""
if self.__runtime_state != state:
self.__runtime_state = state
self.runtimeStateChanged.emit(self.__runtime_state)
def runtime_state(self):
"""
Return the runtime state. This can be `SignalManager.Waiting`
or `SignalManager.Processing`.
"""
return self.__runtime_state
def on_node_removed(self, node):
# remove all pending input signals for node so we don't get
# stale references in process_node.
# NOTE: This does not remove output signals for this node. In
# particular the final 'None' will be delivered to the sink
# nodes even after the source node is no longer in the scheme.
log.info("Node %r removed. Removing pending signals.",
node.title)
self.remove_pending_signals(node)
del self._node_outputs[node]
def on_node_added(self, node):
self._node_outputs[node] = defaultdict(dict)
def link_added(self, link):
# push all current source values to the sink
link.set_runtime_state(SchemeLink.Empty)
if link.enabled:
log.info("Link added (%s). Scheduling signal data update.", link)
self._schedule(self.signals_on_link(link))
self._update()
link.enabled_changed.connect(self.link_enabled_changed)
def link_removed(self, link):
# purge all values in sink's queue
log.info("Link removed (%s). Scheduling signal data purge.", link)
self.purge_link(link)
link.enabled_changed.disconnect(self.link_enabled_changed)
def link_enabled_changed(self, enabled):
if enabled:
link = self.sender()
log.info("Link %s enabled. Scheduling signal data update.", link)
self._schedule(self.signals_on_link(link))
def signals_on_link(self, link):
"""
Return _Signal instances representing the current values
present on the link.
"""
items = self.link_contents(link)
signals = []
for key, value in items.items():
signals.append(_Signal(link, value, key))
return signals
def link_contents(self, link):
"""
Return the contents on link.
"""
node, channel = link.source_node, link.source_channel
if node in self._node_outputs:
return self._node_outputs[node][channel]
else:
# if the the node was already removed it's tracked outputs in
# _node_outputs are cleared, however the final 'None' signal
# deliveries for the link are left in the _input_queue.
pending = [sig for sig in self._input_queue
if sig.link is link]
return {sig.id: sig.value for sig in pending}
def send(self, node, channel, value, id):
"""
"""
log.debug("%r sending %r (id: %r) on channel %r",
node.title, type(value), id, channel.name)
scheme = self.scheme()
self._node_outputs[node][channel][id] = value
links = scheme.find_links(source_node=node, source_channel=channel)
links = filter(is_enabled, links)
signals = []
for link in links:
signals.append(_Signal(link, value, id))
self._schedule(signals)
def purge_link(self, link):
"""
Purge the link (send None for all ids currently present)
"""
contents = self.link_contents(link)
ids = contents.keys()
signals = [_Signal(link, None, id) for id in ids]
self._schedule(signals)
def _schedule(self, signals):
"""
Schedule a list of :class:`_Signal` for delivery.
"""
self._input_queue.extend(signals)
for link in {sig.link for sig in signals}:
# update the SchemeLink's runtime state flags
contents = self.link_contents(link)
if any(value is not None for value in contents.values()):
state = SchemeLink.Active
else:
state = SchemeLink.Empty
link.set_runtime_state(state | SchemeLink.Pending)
if signals:
self.updatesPending.emit()
self._update()
def _update_link(self, link):
"""
Schedule update of a single link.
"""
signals = self.signals_on_link(link)
self._schedule(signals)
def process_queued(self, max_nodes=None):
"""
Process queued signals.
Take one node node from the pending input queue and deliver
all scheduled signals.
"""
if not (max_nodes is None or max_nodes == 1):
warnings.warn(
"`max_nodes` is deprecated and unused (will always equal 1)",
DeprecationWarning, stacklevel=2)
if self.__runtime_state == SignalManager.Processing:
raise RuntimeError("Cannot re-enter 'process_queued'")
if not self._can_process():
raise RuntimeError("Can't process in state %i" % self.__state)
log.info("SignalManager: Processing queued signals")
node_update_front = self.node_update_front()
log.debug("SignalManager: Nodes eligible for update %s",
[node.title for node in node_update_front])
if node_update_front:
node = node_update_front[0]
self._set_runtime_state(SignalManager.Processing)
try:
self.process_node(node)
finally:
self._set_runtime_state(SignalManager.Waiting)
def process_node(self, node):
"""
Process pending input signals for `node`.
"""
signals_in = self.pending_input_signals(node)
self.remove_pending_signals(node)
signals_in = self.compress_signals(signals_in)
log.debug("Processing %r, sending %i signals.",
node.title, len(signals_in))
# Clear the link's pending flag.
for link in {sig.link for sig in signals_in}:
link.set_runtime_state(link.runtime_state() & ~SchemeLink.Pending)
assert ({sig.link for sig in self._input_queue}
.intersection({sig.link for sig in signals_in}) == set([]))
self.processingStarted.emit()
self.processingStarted[SchemeNode].emit(node)
try:
self.send_to_node(node, signals_in)
finally:
self.processingFinished.emit()
self.processingFinished[SchemeNode].emit(node)
def compress_signals(self, signals):
"""
Compress a list of :class:`_Signal` instances to be delivered.
The base implementation returns the list unmodified.
"""
return signals
def send_to_node(self, node, signals):
"""
Abstract. Reimplement in subclass.
Send/notify the :class:`SchemeNode` instance (or whatever
object/instance it is a representation of) that it has new inputs
as represented by the signals list (list of :class:`_Signal`).
"""
raise NotImplementedError
def is_pending(self, node):
"""
Is `node` (class:`SchemeNode`) scheduled for processing (i.e.
it has incoming pending signals).
"""
return node in [signal.link.sink_node for signal in self._input_queue]
def pending_nodes(self):
"""
Return a list of pending nodes.
The nodes are returned in the order they were enqueued for
signal delivery.
Returns
-------
nodes : List[SchemeNode]
"""
return list(unique(sig.link.sink_node for sig in self._input_queue))
def pending_input_signals(self, node):
"""
Return a list of pending input signals for node.
"""
return [signal for signal in self._input_queue
if node is signal.link.sink_node]
def remove_pending_signals(self, node):
"""
Remove pending signals for `node`.
"""
for signal in self.pending_input_signals(node):
try:
self._input_queue.remove(signal)
except ValueError:
pass
def blocking_nodes(self):
"""
Return a list of nodes in a blocking state.
"""
scheme = self.scheme()
return [node for node in scheme.nodes if self.is_blocking(node)]
def is_blocking(self, node):
return False
def node_update_front(self):
"""
Return a list of nodes on the update front, i.e. nodes scheduled for
an update that have no ancestor which is either itself scheduled
for update or is in a blocking state)
.. note::
The node's ancestors are only computed over enabled links.
"""
scheme = self.scheme()
def expand(node):
return [link.sink_node for
link in scheme.find_links(source_node=node) if
link.enabled]
components = strongly_connected_components(scheme.nodes, expand)
node_scc = {node: scc for scc in components for node in scc}
def isincycle(node):
return len(node_scc[node]) > 1
# a list of all nodes currently active/executing a task.
blocking_nodes = set(self.blocking_nodes())
dependents = partial(dependent_nodes, scheme)
blocked_nodes = reduce(set.union,
map(dependents, blocking_nodes),
set(blocking_nodes))
pending = self.pending_nodes()
pending_downstream = set()
for n in pending:
depend = set(dependents(n))
if isincycle(n):
# a pending node in a cycle would would have a circular
# dependency on itself, preventing any progress being made
# by the workflow execution.
cc = node_scc[n]
depend -= set(cc)
pending_downstream.update(depend)
log.debug("Pending nodes: %s", pending)
log.debug("Blocking nodes: %s", blocking_nodes)
noneligible = pending_downstream | blocked_nodes
return [node for node in pending if node not in noneligible]
@Slot()
def __process_next(self):
if not self.__state == SignalManager.Running:
log.debug("Received 'UpdateRequest' while not in 'Running' state")
return
if self.__runtime_state == SignalManager.Processing:
# This happens if someone calls QCoreApplication.processEvents
# from the signal handlers.
# A `__process_next` must be rescheduled when exiting
# process_queued.
log.warning("Received 'UpdateRequest' while in 'process_queued'. "
"An update will be re-scheduled when exiting the "
"current update.")
self.__reschedule = True
return
nbusy = len(self.blocking_nodes())
log.info("'UpdateRequest' event, queued signals: %i, nbusy: %i "
"(MAX_CONCURRENT: %i)",
len(self._input_queue), nbusy, MAX_CONCURRENT)
if self._input_queue and nbusy < MAX_CONCURRENT:
self.process_queued()
if self.__reschedule and self.__state == SignalManager.Running:
self.__reschedule = False
log.debug("Rescheduling signal update")
self.__update_timer.start()
nbusy = len(self.blocking_nodes())
if self.node_update_front() and nbusy < MAX_CONCURRENT:
log.debug("More nodes are eligible for an update. "
"Scheduling another update.")
self._update()
def _update(self):
"""
Schedule processing at a later time.
"""
if self.__state == SignalManager.Running and \
not self.__update_timer.isActive():
self.__update_timer.start()
class OWSelectAttributes(widget.OWWidget):
# pylint: disable=too-many-instance-attributes
name = "Select Columns"
description = "Select columns from the data table and assign them to " \
"data features, classes or meta variables."
icon = "icons/SelectColumns.svg"
priority = 100
keywords = ["filter"]
class Inputs:
data = Input("Data", Table, default=True)
features = Input("Features", AttributeList)
class Outputs:
data = Output("Data", Table)
features = Output("Features", AttributeList, dynamic=False)
want_main_area = False
want_control_area = True
settingsHandler = SelectAttributesDomainContextHandler()
domain_role_hints = ContextSetting({})
use_input_features = Setting(False)
auto_commit = Setting(True)
class Warning(widget.OWWidget.Warning):
mismatching_domain = Msg("Features and data domain do not match")
def __init__(self):
super().__init__()
self.data = None
self.features = None
# Schedule interface updates (enabled buttons) using a coalescing
# single shot timer (complex interactions on selection and filtering
# updates in the 'available_attrs_view')
self.__interface_update_timer = QTimer(self, interval=0, singleShot=True)
self.__interface_update_timer.timeout.connect(
self.__update_interface_state)
# The last view that has the selection for move operation's source
self.__last_active_view = None # type: Optional[QListView]
def update_on_change(view):
# Schedule interface state update on selection change in `view`
self.__last_active_view = view
self.__interface_update_timer.start()
self.controlArea = QWidget(self.controlArea)
self.layout().addWidget(self.controlArea)
layout = QGridLayout()
self.controlArea.setLayout(layout)
layout.setContentsMargins(4, 4, 4, 4)
box = gui.vBox(self.controlArea, "Available Variables",
addToLayout=False)
self.available_attrs = VariablesListItemModel()
filter_edit, self.available_attrs_view = variables_filter(
parent=self, model=self.available_attrs)
box.layout().addWidget(filter_edit)
def dropcompleted(action):
if action == Qt.MoveAction:
self.commit()
self.available_attrs_view.selectionModel().selectionChanged.connect(
partial(update_on_change, self.available_attrs_view))
self.available_attrs_view.dragDropActionDidComplete.connect(dropcompleted)
box.layout().addWidget(self.available_attrs_view)
layout.addWidget(box, 0, 0, 3, 1)
box = gui.vBox(self.controlArea, "Features", addToLayout=False)
self.used_attrs = VariablesListItemModel()
filter_edit, self.used_attrs_view = variables_filter(
parent=self, model=self.used_attrs,
accepted_type=(Orange.data.DiscreteVariable,
Orange.data.ContinuousVariable))
self.used_attrs.rowsInserted.connect(self.__used_attrs_changed)
self.used_attrs.rowsRemoved.connect(self.__used_attrs_changed)
self.used_attrs_view.selectionModel().selectionChanged.connect(
partial(update_on_change, self.used_attrs_view))
self.used_attrs_view.dragDropActionDidComplete.connect(dropcompleted)
self.use_features_box = gui.auto_commit(
self.controlArea, self, "use_input_features",
"Use input features", "Always use input features",
box=False, commit=self.__use_features_clicked,
callback=self.__use_features_changed, addToLayout=False
)
self.enable_use_features_box()
box.layout().addWidget(self.use_features_box)
box.layout().addWidget(filter_edit)
box.layout().addWidget(self.used_attrs_view)
layout.addWidget(box, 0, 2, 1, 1)
box = gui.vBox(self.controlArea, "Target Variable", addToLayout=False)
self.class_attrs = VariablesListItemModel()
self.class_attrs_view = VariablesListItemView(
acceptedType=(Orange.data.DiscreteVariable,
Orange.data.ContinuousVariable))
self.class_attrs_view.setModel(self.class_attrs)
self.class_attrs_view.selectionModel().selectionChanged.connect(
partial(update_on_change, self.class_attrs_view))
self.class_attrs_view.dragDropActionDidComplete.connect(dropcompleted)
self.class_attrs_view.setMaximumHeight(72)
box.layout().addWidget(self.class_attrs_view)
layout.addWidget(box, 1, 2, 1, 1)
box = gui.vBox(self.controlArea, "Meta Attributes", addToLayout=False)
self.meta_attrs = VariablesListItemModel()
self.meta_attrs_view = VariablesListItemView(
acceptedType=Orange.data.Variable)
self.meta_attrs_view.setModel(self.meta_attrs)
self.meta_attrs_view.selectionModel().selectionChanged.connect(
partial(update_on_change, self.meta_attrs_view))
self.meta_attrs_view.dragDropActionDidComplete.connect(dropcompleted)
box.layout().addWidget(self.meta_attrs_view)
layout.addWidget(box, 2, 2, 1, 1)
bbox = gui.vBox(self.controlArea, addToLayout=False, margin=0)
layout.addWidget(bbox, 0, 1, 1, 1)
self.up_attr_button = gui.button(bbox, self, "Up",
callback=partial(self.move_up, self.used_attrs_view))
self.move_attr_button = gui.button(bbox, self, ">",
callback=partial(self.move_selected,
self.used_attrs_view)
)
self.down_attr_button = gui.button(bbox, self, "Down",
callback=partial(self.move_down, self.used_attrs_view))
bbox = gui.vBox(self.controlArea, addToLayout=False, margin=0)
layout.addWidget(bbox, 1, 1, 1, 1)
self.up_class_button = gui.button(bbox, self, "Up",
callback=partial(self.move_up, self.class_attrs_view))
self.move_class_button = gui.button(bbox, self, ">",
callback=partial(self.move_selected,
self.class_attrs_view,
exclusive=False)
)
self.down_class_button = gui.button(bbox, self, "Down",
callback=partial(self.move_down, self.class_attrs_view))
bbox = gui.vBox(self.controlArea, addToLayout=False, margin=0)
layout.addWidget(bbox, 2, 1, 1, 1)
self.up_meta_button = gui.button(bbox, self, "Up",
callback=partial(self.move_up, self.meta_attrs_view))
self.move_meta_button = gui.button(bbox, self, ">",
callback=partial(self.move_selected,
self.meta_attrs_view)
)
self.down_meta_button = gui.button(bbox, self, "Down",
callback=partial(self.move_down, self.meta_attrs_view))
autobox = gui.auto_commit(None, self, "auto_commit", "Send")
layout.addWidget(autobox, 3, 0, 1, 3)
reset = gui.button(None, self, "Reset", callback=self.reset, width=120)
autobox.layout().insertWidget(0, reset)
autobox.layout().insertStretch(1, 20)
layout.setRowStretch(0, 4)
layout.setRowStretch(1, 0)
layout.setRowStretch(2, 2)
layout.setHorizontalSpacing(0)
self.controlArea.setLayout(layout)
self.output_data = None
self.original_completer_items = []
self.resize(600, 600)
@property
def features_from_data_attributes(self):
if self.data is None or self.features is None:
return []
domain = self.data.domain
return [domain[feature.name] for feature in self.features
if feature.name in domain and domain[feature.name]
in domain.attributes]
def can_use_features(self):
return bool(self.features_from_data_attributes) and \
self.features_from_data_attributes != self.used_attrs[:]
def __use_features_changed(self): # Use input features check box
# Needs a check since callback is invoked before object is created
if not hasattr(self, "use_features_box"):
return
self.enable_used_attrs(not self.use_input_features)
if self.use_input_features and self.can_use_features():
self.use_features()
if not self.use_input_features:
self.enable_use_features_box()
def __use_features_clicked(self): # Use input features button
self.use_features()
def __used_attrs_changed(self):
self.enable_use_features_box()
@Inputs.data
def set_data(self, data=None):
self.update_domain_role_hints()
self.closeContext()
self.data = data
if data is not None:
self.openContext(data)
all_vars = data.domain.variables + data.domain.metas
var_sig = lambda attr: (attr.name, vartype(attr))
domain_hints = {var_sig(attr): ("attribute", i)
for i, attr in enumerate(data.domain.attributes)}
domain_hints.update({var_sig(attr): ("meta", i)
for i, attr in enumerate(data.domain.metas)})
if data.domain.class_vars:
domain_hints.update(
{var_sig(attr): ("class", i)
for i, attr in enumerate(data.domain.class_vars)})
# update the hints from context settings
domain_hints.update(self.domain_role_hints)
attrs_for_role = lambda role: [
(domain_hints[var_sig(attr)][1], attr)
for attr in all_vars if domain_hints[var_sig(attr)][0] == role]
attributes = [
attr for place, attr in sorted(attrs_for_role("attribute"),
key=lambda a: a[0])]
classes = [
attr for place, attr in sorted(attrs_for_role("class"),
key=lambda a: a[0])]
metas = [
attr for place, attr in sorted(attrs_for_role("meta"),
key=lambda a: a[0])]
available = [
attr for place, attr in sorted(attrs_for_role("available"),
key=lambda a: a[0])]
self.used_attrs[:] = attributes
self.class_attrs[:] = classes
self.meta_attrs[:] = metas
self.available_attrs[:] = available
else:
self.used_attrs[:] = []
self.class_attrs[:] = []
self.meta_attrs[:] = []
self.available_attrs[:] = []
def update_domain_role_hints(self):
""" Update the domain hints to be stored in the widgets settings.
"""
hints_from_model = lambda role, model: [
((attr.name, vartype(attr)), (role, i))
for i, attr in enumerate(model)]
hints = dict(hints_from_model("available", self.available_attrs))
hints.update(hints_from_model("attribute", self.used_attrs))
hints.update(hints_from_model("class", self.class_attrs))
hints.update(hints_from_model("meta", self.meta_attrs))
self.domain_role_hints = hints
@Inputs.features
def set_features(self, features):
self.features = features
def handleNewSignals(self):
self.check_data()
self.enable_used_attrs()
self.enable_use_features_box()
if self.use_input_features and len(self.features_from_data_attributes):
self.enable_used_attrs(False)
self.use_features()
self.unconditional_commit()
def check_data(self):
self.Warning.mismatching_domain.clear()
if self.data is not None and self.features is not None and \
not len(self.features_from_data_attributes):
self.Warning.mismatching_domain()
def enable_used_attrs(self, enable=True):
self.up_attr_button.setEnabled(enable)
self.move_attr_button.setEnabled(enable)
self.down_attr_button.setEnabled(enable)
self.used_attrs_view.setEnabled(enable)
self.used_attrs_view.repaint()
def enable_use_features_box(self):
self.use_features_box.button.setEnabled(self.can_use_features())
enable_checkbox = bool(self.features_from_data_attributes)
self.use_features_box.setHidden(not enable_checkbox)
self.use_features_box.repaint()
def use_features(self):
attributes = self.features_from_data_attributes
available, used = self.available_attrs[:], self.used_attrs[:]
self.available_attrs[:] = [attr for attr in used + available
if attr not in attributes]
self.used_attrs[:] = attributes
self.commit()
def selected_rows(self, view):
""" Return the selected rows in the view.
"""
rows = view.selectionModel().selectedRows()
model = view.model()
if isinstance(model, QSortFilterProxyModel):
rows = [model.mapToSource(r) for r in rows]
return [r.row() for r in rows]
def move_rows(self, view, rows, offset):
model = view.model()
newrows = [min(max(0, row + offset), len(model) - 1) for row in rows]
for row, newrow in sorted(zip(rows, newrows), reverse=offset > 0):
model[row], model[newrow] = model[newrow], model[row]
selection = QItemSelection()
for nrow in newrows:
index = model.index(nrow, 0)
selection.select(index, index)
view.selectionModel().select(
selection, QItemSelectionModel.ClearAndSelect)
self.commit()
def move_up(self, view):
selected = self.selected_rows(view)
self.move_rows(view, selected, -1)
def move_down(self, view):
selected = self.selected_rows(view)
self.move_rows(view, selected, 1)
def move_selected(self, view, exclusive=False):
if self.selected_rows(view):
self.move_selected_from_to(view, self.available_attrs_view)
elif self.selected_rows(self.available_attrs_view):
self.move_selected_from_to(self.available_attrs_view, view,
exclusive)
def move_selected_from_to(self, src, dst, exclusive=False):
self.move_from_to(src, dst, self.selected_rows(src), exclusive)
def move_from_to(self, src, dst, rows, exclusive=False):
src_model = source_model(src)
attrs = [src_model[r] for r in rows]
for s1, s2 in reversed(list(slices(rows))):
del src_model[s1:s2]
dst_model = source_model(dst)
dst_model.extend(attrs)
self.commit()
def __update_interface_state(self):
last_view = self.__last_active_view
if last_view is not None:
self.update_interface_state(last_view)
def update_interface_state(self, focus=None, selected=None, deselected=None):
for view in [self.available_attrs_view, self.used_attrs_view,
self.class_attrs_view, self.meta_attrs_view]:
if view is not focus and not view.hasFocus() \
and view.selectionModel().hasSelection():
view.selectionModel().clear()
def selected_vars(view):
model = source_model(view)
return [model[i] for i in self.selected_rows(view)]
available_selected = selected_vars(self.available_attrs_view)
attrs_selected = selected_vars(self.used_attrs_view)
class_selected = selected_vars(self.class_attrs_view)
meta_selected = selected_vars(self.meta_attrs_view)
available_types = set(map(type, available_selected))
all_primitive = all(var.is_primitive()
for var in available_types)
move_attr_enabled = \
((available_selected and all_primitive) or attrs_selected) and \
self.used_attrs_view.isEnabled()
self.move_attr_button.setEnabled(bool(move_attr_enabled))
if move_attr_enabled:
self.move_attr_button.setText(">" if available_selected else "<")
move_class_enabled = (all_primitive and available_selected) or class_selected
self.move_class_button.setEnabled(bool(move_class_enabled))
if move_class_enabled:
self.move_class_button.setText(">" if available_selected else "<")
move_meta_enabled = available_selected or meta_selected
self.move_meta_button.setEnabled(bool(move_meta_enabled))
if move_meta_enabled:
self.move_meta_button.setText(">" if available_selected else "<")
self.__last_active_view = None
self.__interface_update_timer.stop()
def commit(self):
self.update_domain_role_hints()
if self.data is not None:
attributes = list(self.used_attrs)
class_var = list(self.class_attrs)
metas = list(self.meta_attrs)
domain = Orange.data.Domain(attributes, class_var, metas)
newdata = self.data.transform(domain)
self.output_data = newdata
self.Outputs.data.send(newdata)
self.Outputs.features.send(AttributeList(attributes))
else:
self.output_data = None
self.Outputs.data.send(None)
self.Outputs.features.send(None)
def reset(self):
self.enable_used_attrs()
self.use_features_box.checkbox.setChecked(False)
if self.data is not None:
self.available_attrs[:] = []
self.used_attrs[:] = self.data.domain.attributes
self.class_attrs[:] = self.data.domain.class_vars
self.meta_attrs[:] = self.data.domain.metas
self.update_domain_role_hints()
self.commit()
def send_report(self):
if not self.data or not self.output_data:
return
in_domain, out_domain = self.data.domain, self.output_data.domain
self.report_domain("Input data", self.data.domain)
if (in_domain.attributes, in_domain.class_vars, in_domain.metas) == (
out_domain.attributes, out_domain.class_vars, out_domain.metas):
self.report_paragraph("Output data", "No changes.")
else:
self.report_domain("Output data", self.output_data.domain)
diff = list(set(in_domain.variables + in_domain.metas) -
set(out_domain.variables + out_domain.metas))
if diff:
text = "%i (%s)" % (len(diff), ", ".join(x.name for x in diff))
self.report_items((("Removed", text),))
class OWMDS(OWWidget):
name = "MDS"
description = "Two-dimensional data projection by multidimensional " \
"scaling constructed from a distance matrix."
icon = "icons/MDS.svg"
class Inputs:
data = Input("Data", Orange.data.Table, default=True)
distances = Input("Distances", Orange.misc.DistMatrix)
data_subset = Input("Data Subset", Orange.data.Table)
class Outputs:
selected_data = Output("Selected Data", Orange.data.Table, default=True)
annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME, Orange.data.Table)
settings_version = 2
#: Initialization type
PCA, Random = 0, 1
#: Refresh rate
RefreshRate = [
("Every iteration", 1),
("Every 5 steps", 5),
("Every 10 steps", 10),
("Every 25 steps", 25),
("Every 50 steps", 50),
("None", -1)
]
#: Runtime state
Running, Finished, Waiting = 1, 2, 3
settingsHandler = settings.DomainContextHandler()
max_iter = settings.Setting(300)
initialization = settings.Setting(PCA)
refresh_rate = settings.Setting(3)
# output embedding role.
NoRole, AttrRole, AddAttrRole, MetaRole = 0, 1, 2, 3
auto_commit = settings.Setting(True)
selection_indices = settings.Setting(None, schema_only=True)
#: Percentage of all pairs displayed (ranges from 0 to 20)
connected_pairs = settings.Setting(5)
legend_anchor = settings.Setting(((1, 0), (1, 0)))
graph = SettingProvider(OWMDSGraph)
jitter_sizes = [0, 0.1, 0.5, 1, 2, 3, 4, 5, 7, 10]
graph_name = "graph.plot_widget.plotItem"
class Error(OWWidget.Error):
not_enough_rows = Msg("Input data needs at least 2 rows")
matrix_too_small = Msg("Input matrix must be at least 2x2")
no_attributes = Msg("Data has no attributes")
mismatching_dimensions = \
Msg("Data and distances dimensions do not match.")
out_of_memory = Msg("Out of memory")
optimization_error = Msg("Error during optimization\n{}")
def __init__(self):
super().__init__()
#: Input dissimilarity matrix
self.matrix = None # type: Optional[Orange.misc.DistMatrix]
#: Effective data used for plot styling/annotations. Can be from the
#: input signal (`self.signal_data`) or the input matrix
#: (`self.matrix.data`)
self.data = None # type: Optional[Orange.data.Table]
#: Input subset data table
self.subset_data = None # type: Optional[Orange.data.Table]
#: Data table from the `self.matrix.row_items` (if present)
self.matrix_data = None # type: Optional[Orange.data.Table]
#: Input data table
self.signal_data = None
self._similar_pairs = None
self._subset_mask = None # type: Optional[np.ndarray]
self._invalidated = False
self.effective_matrix = None
self._curve = None
self.variable_x = ContinuousVariable("mds-x")
self.variable_y = ContinuousVariable("mds-y")
self.__update_loop = None
# timer for scheduling updates
self.__timer = QTimer(self, singleShot=True, interval=0)
self.__timer.timeout.connect(self.__next_step)
self.__state = OWMDS.Waiting
self.__in_next_step = False
self.__draw_similar_pairs = False
box = gui.vBox(self.controlArea, "MDS Optimization")
form = QFormLayout(
labelAlignment=Qt.AlignLeft,
formAlignment=Qt.AlignLeft,
fieldGrowthPolicy=QFormLayout.AllNonFixedFieldsGrow,
verticalSpacing=10
)
form.addRow(
"Max iterations:",
gui.spin(box, self, "max_iter", 10, 10 ** 4, step=1))
form.addRow(
"Initialization:",
gui.radioButtons(box, self, "initialization", btnLabels=("PCA (Torgerson)", "Random"),
callback=self.__invalidate_embedding))
box.layout().addLayout(form)
form.addRow(
"Refresh:",
gui.comboBox(box, self, "refresh_rate", items=[t for t, _ in OWMDS.RefreshRate],
callback=self.__invalidate_refresh))
gui.separator(box, 10)
self.runbutton = gui.button(box, self, "Run", callback=self._toggle_run)
box = gui.vBox(self.mainArea, True, margin=0)
self.graph = OWMDSGraph(self, box, "MDSGraph", view_box=MDSInteractiveViewBox)
box.layout().addWidget(self.graph.plot_widget)
self.plot = self.graph.plot_widget
g = self.graph.gui
box = g.point_properties_box(self.controlArea)
self.models = g.points_models
self.size_model = self.models[2]
self.label_model = self.models[3]
self.size_model.order = \
self.size_model.order[:1] + ("Stress", ) + self.models[2].order[1:]
gui.hSlider(box, self, "connected_pairs", label="Show similar pairs:", minValue=0,
maxValue=20, createLabel=False, callback=self._on_connected_changed)
g.add_widgets(ids=[g.JitterSizeSlider], widget=box)
box = gui.vBox(self.controlArea, "Plot Properties")
g.add_widgets([g.ShowLegend,
g.ToolTipShowsAll,
g.ClassDensity,
g.LabelOnlySelected], box)
self.controlArea.layout().addStretch(100)
self.icons = gui.attributeIconDict
palette = self.graph.plot_widget.palette()
self.graph.set_palette(palette)
gui.rubber(self.controlArea)
self.graph.box_zoom_select(self.controlArea)
gui.auto_commit(box, self, "auto_commit", "Send Selected",
checkbox_label="Send selected automatically",
box=None)
self.plot.getPlotItem().hideButtons()
self.plot.setRenderHint(QPainter.Antialiasing)
self.graph.jitter_continuous = True
self._initialize()
def reset_graph_data(self, *_):
if self.data is not None:
self.graph.rescale_data()
self.update_graph()
self.connect_pairs()
def update_colors(self):
pass
def update_density(self):
self.update_graph(reset_view=False)
def update_regression_line(self):
self.update_graph(reset_view=False)
def init_attr_values(self):
self.graph.set_domain(self.data)
def prepare_data(self):
pass
def update_graph(self, reset_view=True, **_):
self.graph.zoomStack = []
if self.graph.data is None:
return
self.graph.update_data(self.variable_x, self.variable_y, True)
def selection_changed(self):
self.commit()
@Inputs.data
@check_sql_input
def set_data(self, data):
"""Set the input dataset.
Parameters
----------
data : Optional[Orange.data.Table]
"""
if data is not None and len(data) < 2:
self.Error.not_enough_rows()
data = None
else:
self.Error.not_enough_rows.clear()
self.signal_data = data
if self.matrix is not None and data is not None and len(self.matrix) == len(data):
self.closeContext()
self.data = data
self.init_attr_values()
self.openContext(data)
else:
self._invalidated = True
@Inputs.distances
def set_disimilarity(self, matrix):
"""Set the dissimilarity (distance) matrix.
Parameters
----------
matrix : Optional[Orange.misc.DistMatrix]
"""
if matrix is not None and len(matrix) < 2:
self.Error.matrix_too_small()
matrix = None
else:
self.Error.matrix_too_small.clear()
self.matrix = matrix
self.matrix_data = matrix.row_items if matrix is not None else None
self._invalidated = True
@Inputs.data_subset
def set_subset_data(self, subset_data):
"""Set a subset of `data` input to highlight in the plot.
Parameters
----------
subset_data: Optional[Orange.data.Table]
"""
self.subset_data = subset_data
# invalidate the pen/brush when the subset is changed
self._subset_mask = None # type: Optional[np.ndarray]
self.controls.graph.alpha_value.setEnabled(subset_data is None)
def _clear(self):
self._similar_pairs = None
self.__set_update_loop(None)
self.__state = OWMDS.Waiting
def _clear_plot(self):
self.graph.plot_widget.clear()
def _initialize(self):
# clear everything
self.closeContext()
self._clear()
self.Error.clear()
self.data = None
self.effective_matrix = None
self.embedding = None
self.init_attr_values()
# if no data nor matrix is present reset plot
if self.signal_data is None and self.matrix is None:
return
if self.signal_data is not None and self.matrix is not None and \
len(self.signal_data) != len(self.matrix):
self.Error.mismatching_dimensions()
self._update_plot()
return
if self.signal_data is not None:
self.data = self.signal_data
elif self.matrix_data is not None:
self.data = self.matrix_data
if self.matrix is not None:
self.effective_matrix = self.matrix
if self.matrix.axis == 0 and self.data is self.matrix_data:
self.data = None
elif self.data.domain.attributes:
preprocessed_data = Orange.projection.MDS().preprocess(self.data)
self.effective_matrix = Orange.distance.Euclidean(preprocessed_data)
else:
self.Error.no_attributes()
return
self.init_attr_values()
self.openContext(self.data)
def _toggle_run(self):
if self.__state == OWMDS.Running:
self.stop()
self._invalidate_output()
else:
self.start()
def start(self):
if self.__state == OWMDS.Running:
return
elif self.__state == OWMDS.Finished:
# Resume/continue from a previous run
self.__start()
elif self.__state == OWMDS.Waiting and \
self.effective_matrix is not None:
self.__start()
def stop(self):
if self.__state == OWMDS.Running:
self.__set_update_loop(None)
def __start(self):
self.__draw_similar_pairs = False
X = self.effective_matrix
init = self.embedding
# number of iterations per single GUI update step
_, step_size = OWMDS.RefreshRate[self.refresh_rate]
if step_size == -1:
step_size = self.max_iter
def update_loop(X, max_iter, step, init):
"""
return an iterator over successive improved MDS point embeddings.
"""
# NOTE: this code MUST NOT call into QApplication.processEvents
done = False
iterations_done = 0
oldstress = np.finfo(np.float).max
init_type = "PCA" if self.initialization == OWMDS.PCA else "random"
while not done:
step_iter = min(max_iter - iterations_done, step)
mds = Orange.projection.MDS(
dissimilarity="precomputed", n_components=2,
n_init=1, max_iter=step_iter,
init_type=init_type, init_data=init)
mdsfit = mds(X)
iterations_done += step_iter
embedding, stress = mdsfit.embedding_, mdsfit.stress_
stress /= np.sqrt(np.sum(embedding ** 2, axis=1)).sum()
if iterations_done >= max_iter:
done = True
elif (oldstress - stress) < mds.params["eps"]:
done = True
init = embedding
oldstress = stress
yield embedding, mdsfit.stress_, iterations_done / max_iter
self.__set_update_loop(update_loop(X, self.max_iter, step_size, init))
self.progressBarInit(processEvents=None)
def __set_update_loop(self, loop):
"""
Set the update `loop` coroutine.
The `loop` is a generator yielding `(embedding, stress, progress)`
tuples where `embedding` is a `(N, 2) ndarray` of current updated
MDS points, `stress` is the current stress and `progress` a float
ratio (0 <= progress <= 1)
If an existing update coroutine loop is already in place it is
interrupted (i.e. closed).
.. note::
The `loop` must not explicitly yield control flow to the event
loop (i.e. call `QApplication.processEvents`)
"""
if self.__update_loop is not None:
self.__update_loop.close()
self.__update_loop = None
self.progressBarFinished(processEvents=None)
self.__update_loop = loop
if loop is not None:
self.setBlocking(True)
self.progressBarInit(processEvents=None)
self.setStatusMessage("Running")
self.runbutton.setText("Stop")
self.__state = OWMDS.Running
self.__timer.start()
else:
self.setBlocking(False)
self.setStatusMessage("")
self.runbutton.setText("Start")
self.__state = OWMDS.Finished
self.__timer.stop()
def __next_step(self):
if self.__update_loop is None:
return
assert not self.__in_next_step
self.__in_next_step = True
loop = self.__update_loop
self.Error.out_of_memory.clear()
try:
embedding, _, progress = next(self.__update_loop)
assert self.__update_loop is loop
except StopIteration:
self.__set_update_loop(None)
self.unconditional_commit()
self.__draw_similar_pairs = True
self._update_plot()
except MemoryError:
self.Error.out_of_memory()
self.__set_update_loop(None)
self.__draw_similar_pairs = True
except Exception as exc:
self.Error.optimization_error(str(exc))
self.__set_update_loop(None)
self.__draw_similar_pairs = True
else:
self.progressBarSet(100.0 * progress, processEvents=None)
self.embedding = embedding
self._update_plot()
# schedule next update
self.__timer.start()
self.__in_next_step = False
def __invalidate_embedding(self):
# reset/invalidate the MDS embedding, to the default initialization
# (Random or PCA), restarting the optimization if necessary.
if self.embedding is None:
return
state = self.__state
if self.__update_loop is not None:
self.__set_update_loop(None)
X = self.effective_matrix
if self.initialization == OWMDS.PCA:
self.embedding = torgerson(X)
else:
self.embedding = np.random.rand(len(X), 2)
self._update_plot()
# restart the optimization if it was interrupted.
if state == OWMDS.Running:
self.__start()
def __invalidate_refresh(self):
state = self.__state
if self.__update_loop is not None:
self.__set_update_loop(None)
# restart the optimization if it was interrupted.
# TODO: decrease the max iteration count by the already
# completed iterations count.
if state == OWMDS.Running:
self.__start()
def handleNewSignals(self):
if self._invalidated:
self.__draw_similar_pairs = False
self._invalidated = False
self._initialize()
self.start()
if self._subset_mask is None and self.subset_data is not None and \
self.data is not None:
self._subset_mask = np.in1d(self.data.ids, self.subset_data.ids)
self._update_plot(new=True)
self.unconditional_commit()
def _invalidate_output(self):
self.commit()
def _on_connected_changed(self):
self._similar_pairs = None
self.connect_pairs()
def _update_plot(self, new=False):
self._clear_plot()
if self.embedding is not None:
self._setup_plot(new=new)
else:
self.graph.new_data(None)
def connect_pairs(self):
if self._curve:
self.graph.plot_widget.removeItem(self._curve)
if not (self.connected_pairs and self.__draw_similar_pairs):
return
emb_x, emb_y = self.graph.get_xy_data_positions(
self.variable_x, self.variable_y, self.graph.valid_data)
if self._similar_pairs is None:
# This code requires storing lower triangle of X (n x n / 2
# doubles), n x n / 2 * 2 indices to X, n x n / 2 indices for
# argsort result. If this becomes an issue, it can be reduced to
# n x n argsort indices by argsorting the entire X. Then we
# take the first n + 2 * p indices. We compute their coordinates
# i, j in the original matrix. We keep those for which i < j.
# n + 2 * p will suffice to exclude the diagonal (i = j). If the
# number of those for which i < j is smaller than p, we instead
# take i > j. Among those that remain, we take the first p.
# Assuming that MDS can't show so many points that memory could
# become an issue, I preferred using simpler code.
m = self.effective_matrix
n = len(m)
p = min(n * (n - 1) // 2 * self.connected_pairs // 100,
MAX_N_PAIRS * self.connected_pairs // 20)
indcs = np.triu_indices(n, 1)
sorted = np.argsort(m[indcs])[:p]
self._similar_pairs = fpairs = np.empty(2 * p, dtype=int)
fpairs[::2] = indcs[0][sorted]
fpairs[1::2] = indcs[1][sorted]
emb_x_pairs = emb_x[self._similar_pairs].reshape((-1, 2))
emb_y_pairs = emb_y[self._similar_pairs].reshape((-1, 2))
# Filter out zero distance lines (in embedding coords).
# Null (zero length) line causes bad rendering artifacts
# in Qt when using the raster graphics system (see gh-issue: 1668).
(x1, x2), (y1, y2) = (emb_x_pairs.T, emb_y_pairs.T)
pairs_mask = ~(np.isclose(x1, x2) & np.isclose(y1, y2))
emb_x_pairs = emb_x_pairs[pairs_mask, :]
emb_y_pairs = emb_y_pairs[pairs_mask, :]
self._curve = pg.PlotCurveItem(
emb_x_pairs.ravel(), emb_y_pairs.ravel(),
pen=pg.mkPen(0.8, width=2, cosmetic=True),
connect="pairs", antialias=True)
self.graph.plot_widget.addItem(self._curve)
def _setup_plot(self, new=False):
emb_x, emb_y = self.embedding[:, 0], self.embedding[:, 1]
coords = np.vstack((emb_x, emb_y)).T
data = self.data
attributes = data.domain.attributes + (self.variable_x, self.variable_y)
domain = Domain(attributes=attributes,
class_vars=data.domain.class_vars,
metas=data.domain.metas)
data = Table.from_numpy(domain, X=hstack((data.X, coords)),
Y=data.Y, metas=data.metas)
subset_data = data[self._subset_mask] if self._subset_mask is not None else None
self.graph.new_data(data, subset_data=subset_data, new=new)
self.graph.update_data(self.variable_x, self.variable_y, True)
self.connect_pairs()
def commit(self):
if self.embedding is not None:
names = get_unique_names([v.name for v in self.data.domain.variables],
["mds-x", "mds-y"])
output = embedding = Orange.data.Table.from_numpy(
Orange.data.Domain([ContinuousVariable(names[0]), ContinuousVariable(names[1])]),
self.embedding
)
else:
output = embedding = None
if self.embedding is not None and self.data is not None:
domain = self.data.domain
domain = Orange.data.Domain(domain.attributes,
domain.class_vars,
domain.metas + embedding.domain.attributes)
output = self.data.transform(domain)
output.metas[:, -2:] = embedding.X
selection = self.graph.get_selection()
if output is not None and len(selection) > 0:
selected = output[selection]
else:
selected = None
if self.graph.selection is not None and np.max(self.graph.selection) > 1:
annotated = create_groups_table(output, self.graph.selection)
else:
annotated = create_annotated_table(output, selection)
self.Outputs.selected_data.send(selected)
self.Outputs.annotated_data.send(annotated)
def onDeleteWidget(self):
super().onDeleteWidget()
self._clear_plot()
self._clear()
def send_report(self):
if self.data is None:
return
def name(var):
return var and var.name
caption = report.render_items_vert((
("Color", name(self.graph.attr_color)),
("Label", name(self.graph.attr_label)),
("Shape", name(self.graph.attr_shape)),
("Size", name(self.graph.attr_size)),
("Jittering", self.graph.jitter_size != 0 and "{} %".format(self.graph.jitter_size))))
self.report_plot()
if caption:
self.report_caption(caption)
@classmethod
def migrate_settings(cls, settings_, version):
if version < 2:
settings_graph = {}
for old, new in (("label_only_selected", "label_only_selected"),
("symbol_opacity", "alpha_value"),
("symbol_size", "point_width"),
("jitter", "jitter_size")):
settings_graph[new] = settings_[old]
settings_["graph"] = settings_graph
settings_["auto_commit"] = settings_["autocommit"]
@classmethod
def migrate_context(cls, context, version):
if version < 2:
domain = context.ordered_domain
n_domain = [t for t in context.ordered_domain if t[1] == 2]
c_domain = [t for t in context.ordered_domain if t[1] == 1]
context_values_graph = {}
for _, old_val, new_val in ((domain, "color_value", "attr_color"),
(c_domain, "shape_value", "attr_shape"),
(n_domain, "size_value", "attr_size"),
(domain, "label_value", "attr_label")):
tmp = context.values[old_val]
if tmp[1] >= 0:
context_values_graph[new_val] = (tmp[0], tmp[1] + 100)
elif tmp[0] != "Stress":
context_values_graph[new_val] = None
else:
context_values_graph[new_val] = tmp
context.values["graph"] = context_values_graph
class OWScatterPlot(OWDataProjectionWidget):
"""Scatterplot visualization with explorative analysis and intelligent
data visualization enhancements."""
name = 'Scatter Plot'
description = "Interactive scatter plot visualization with " \
"intelligent data visualization enhancements."
icon = "icons/ScatterPlot.svg"
priority = 140
keywords = []
class Inputs(OWDataProjectionWidget.Inputs):
features = Input("Features", AttributeList)
class Outputs(OWDataProjectionWidget.Outputs):
features = Output("Features", AttributeList, dynamic=False)
settings_version = 3
auto_sample = Setting(True)
attr_x = ContextSetting(None)
attr_y = ContextSetting(None)
tooltip_shows_all = Setting(True)
GRAPH_CLASS = OWScatterPlotGraph
graph = SettingProvider(OWScatterPlotGraph)
embedding_variables_names = None
class Warning(OWDataProjectionWidget.Warning):
missing_coords = Msg("Plot cannot be displayed because '{}' or '{}' "
"is missing for all data points")
class Information(OWDataProjectionWidget.Information):
sampled_sql = Msg("Large SQL table; showing a sample.")
missing_coords = Msg(
"Points with missing '{}' or '{}' are not displayed")
def __init__(self):
self.sql_data = None # Orange.data.sql.table.SqlTable
self.attribute_selection_list = None # list of Orange.data.Variable
self.__timer = QTimer(self, interval=1200)
self.__timer.timeout.connect(self.add_data)
super().__init__()
# manually register Matplotlib file writers
self.graph_writers = self.graph_writers.copy()
for w in [MatplotlibFormat, MatplotlibPDFFormat]:
for ext in w.EXTENSIONS:
self.graph_writers[ext] = w
def _add_controls(self):
self._add_controls_axis()
self._add_controls_sampling()
super()._add_controls()
self.gui.add_widget(self.gui.JitterNumericValues, self._effects_box)
self.gui.add_widgets([
self.gui.ShowGridLines, self.gui.ToolTipShowsAll,
self.gui.RegressionLine
], self._plot_box)
def _add_controls_axis(self):
common_options = dict(labelWidth=50,
orientation=Qt.Horizontal,
sendSelectedValue=True,
valueType=str,
contentsLength=14)
box = gui.vBox(self.controlArea, True)
dmod = DomainModel
self.xy_model = DomainModel(dmod.MIXED, valid_types=dmod.PRIMITIVE)
self.cb_attr_x = gui.comboBox(box,
self,
"attr_x",
label="Axis x:",
callback=self.attr_changed,
model=self.xy_model,
**common_options)
self.cb_attr_y = gui.comboBox(box,
self,
"attr_y",
label="Axis y:",
callback=self.attr_changed,
model=self.xy_model,
**common_options)
vizrank_box = gui.hBox(box)
self.vizrank, self.vizrank_button = ScatterPlotVizRank.add_vizrank(
vizrank_box, self, "Find Informative Projections", self.set_attr)
def _add_controls_sampling(self):
self.sampling = gui.auto_commit(self.controlArea,
self,
"auto_sample",
"Sample",
box="Sampling",
callback=self.switch_sampling,
commit=lambda: self.add_data(1))
self.sampling.setVisible(False)
@property
def effective_variables(self):
return [self.attr_x, self.attr_y]
def _vizrank_color_change(self):
self.vizrank.initialize()
is_enabled = self.data is not None and not self.data.is_sparse() and \
len(self.xy_model) > 2 and len(self.data[self.valid_data]) > 1 \
and np.all(np.nan_to_num(np.nanstd(self.data.X, 0)) != 0)
self.vizrank_button.setEnabled(
is_enabled and self.attr_color is not None and not np.isnan(
self.data.get_column_view(
self.attr_color)[0].astype(float)).all())
text = "Color variable has to be selected." \
if is_enabled and self.attr_color is None else ""
self.vizrank_button.setToolTip(text)
def set_data(self, data):
if self.data and data and self.data.checksum() == data.checksum():
return
super().set_data(data)
def findvar(name, iterable):
"""Find a Orange.data.Variable in `iterable` by name"""
for el in iterable:
if isinstance(el, Variable) and el.name == name:
return el
return None
# handle restored settings from < 3.3.9 when attr_* were stored
# by name
if isinstance(self.attr_x, str):
self.attr_x = findvar(self.attr_x, self.xy_model)
if isinstance(self.attr_y, str):
self.attr_y = findvar(self.attr_y, self.xy_model)
if isinstance(self.attr_label, str):
self.attr_label = findvar(self.attr_label, self.gui.label_model)
if isinstance(self.attr_color, str):
self.attr_color = findvar(self.attr_color, self.gui.color_model)
if isinstance(self.attr_shape, str):
self.attr_shape = findvar(self.attr_shape, self.gui.shape_model)
if isinstance(self.attr_size, str):
self.attr_size = findvar(self.attr_size, self.gui.size_model)
def check_data(self):
self.clear_messages()
self.__timer.stop()
self.sampling.setVisible(False)
self.sql_data = None
if isinstance(self.data, SqlTable):
if self.data.approx_len() < 4000:
self.data = Table(self.data)
else:
self.Information.sampled_sql()
self.sql_data = self.data
data_sample = self.data.sample_time(0.8, no_cache=True)
data_sample.download_data(2000, partial=True)
self.data = Table(data_sample)
self.sampling.setVisible(True)
if self.auto_sample:
self.__timer.start()
if self.data is not None and (len(self.data) == 0
or len(self.data.domain) == 0):
self.data = None
def get_embedding(self):
self.valid_data = None
if self.data is None:
return None
x_data = self.get_column(self.attr_x, filter_valid=False)
y_data = self.get_column(self.attr_y, filter_valid=False)
if x_data is None or y_data is None:
return None
self.Warning.missing_coords.clear()
self.Information.missing_coords.clear()
self.valid_data = np.isfinite(x_data) & np.isfinite(y_data)
if self.valid_data is not None and not np.all(self.valid_data):
msg = self.Information if np.any(self.valid_data) else self.Warning
msg.missing_coords(self.attr_x.name, self.attr_y.name)
return np.vstack((x_data, y_data)).T
# Tooltip
def _point_tooltip(self, point_id, skip_attrs=()):
point_data = self.data[point_id]
xy_attrs = (self.attr_x, self.attr_y)
text = "<br/>".join(
escape('{} = {}'.format(var.name, point_data[var]))
for var in xy_attrs)
if self.tooltip_shows_all:
others = super()._point_tooltip(point_id, skip_attrs=xy_attrs)
if others:
text = "<b>{}</b><br/><br/>{}".format(text, others)
return text
def can_draw_regresssion_line(self):
return self.data is not None and\
self.data.domain is not None and \
self.attr_x.is_continuous and \
self.attr_y.is_continuous
def add_data(self, time=0.4):
if self.data and len(self.data) > 2000:
self.__timer.stop()
return
data_sample = self.sql_data.sample_time(time, no_cache=True)
if data_sample:
data_sample.download_data(2000, partial=True)
data = Table(data_sample)
self.data = Table.concatenate((self.data, data), axis=0)
self.handleNewSignals()
def init_attr_values(self):
super().init_attr_values()
data = self.data
domain = data.domain if data and len(data) else None
self.xy_model.set_domain(domain)
self.attr_x = self.xy_model[0] if self.xy_model else None
self.attr_y = self.xy_model[1] if len(self.xy_model) >= 2 \
else self.attr_x
def switch_sampling(self):
self.__timer.stop()
if self.auto_sample and self.sql_data:
self.add_data()
self.__timer.start()
def set_subset_data(self, subset_data):
self.warning()
if isinstance(subset_data, SqlTable):
if subset_data.approx_len() < AUTO_DL_LIMIT:
subset_data = Table(subset_data)
else:
self.warning("Data subset does not support large Sql tables")
subset_data = None
super().set_subset_data(subset_data)
# called when all signals are received, so the graph is updated only once
def handleNewSignals(self):
if self.attribute_selection_list and self.data is not None and \
self.data.domain is not None and \
all(attr in self.data.domain for attr
in self.attribute_selection_list):
self.set_attr(self.attribute_selection_list[0],
self.attribute_selection_list[1])
self.attribute_selection_list = None
else:
super().handleNewSignals()
self._vizrank_color_change()
self.cb_reg_line.setEnabled(self.can_draw_regresssion_line())
@Inputs.features
def set_shown_attributes(self, attributes):
if attributes and len(attributes) >= 2:
self.attribute_selection_list = attributes[:2]
else:
self.attribute_selection_list = None
def set_attr(self, attr_x, attr_y):
if attr_x != self.attr_x or attr_y != self.attr_y:
self.attr_x, self.attr_y = attr_x, attr_y
self.attr_changed()
def attr_changed(self):
self.cb_reg_line.setEnabled(self.can_draw_regresssion_line())
self.setup_plot()
self.commit()
def setup_plot(self):
super().setup_plot()
for axis, var in (("bottom", self.attr_x), ("left", self.attr_y)):
self.graph.set_axis_title(axis, var)
if var and var.is_discrete:
self.graph.set_axis_labels(axis,
get_variable_values_sorted(var))
else:
self.graph.set_axis_labels(axis, None)
def colors_changed(self):
super().colors_changed()
self._vizrank_color_change()
def commit(self):
super().commit()
self.send_features()
def send_features(self):
features = [attr for attr in [self.attr_x, self.attr_y] if attr]
self.Outputs.features.send(features or None)
def get_widget_name_extension(self):
if self.data is not None:
return "{} vs {}".format(self.attr_x.name, self.attr_y.name)
return None
def _get_send_report_caption(self):
return report.render_items_vert(
(("Color", self._get_caption_var_name(self.attr_color)),
("Label", self._get_caption_var_name(self.attr_label)),
("Shape", self._get_caption_var_name(self.attr_shape)),
("Size", self._get_caption_var_name(self.attr_size)),
("Jittering", (self.attr_x.is_discrete or self.attr_y.is_discrete
or self.graph.jitter_continuous)
and self.graph.jitter_size)))
@classmethod
def migrate_settings(cls, settings, version):
if version < 2 and "selection" in settings and settings["selection"]:
settings["selection_group"] = [(a, 1)
for a in settings["selection"]]
if version < 3:
if "auto_send_selection" in settings:
settings["auto_commit"] = settings["auto_send_selection"]
if "selection_group" in settings:
settings["selection"] = settings["selection_group"]
@classmethod
def migrate_context(cls, context, version):
if version < 3:
values = context.values
values["attr_color"] = values["graph"]["attr_color"]
values["attr_size"] = values["graph"]["attr_size"]
values["attr_shape"] = values["graph"]["attr_shape"]
values["attr_label"] = values["graph"]["attr_label"]
class OWScatterPlot(OWWidget):
"""Scatterplot visualization with explorative analysis and intelligent
data visualization enhancements."""
name = 'Scatter Plot'
description = "Interactive scatter plot visualization with " \
"intelligent data visualization enhancements."
icon = "icons/ScatterPlot.svg"
priority = 140
class Inputs:
data = Input("Data", Table, default=True)
data_subset = Input("Data Subset", Table)
features = Input("Features", AttributeList)
class Outputs:
selected_data = Output("Selected Data", Table, default=True)
annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME, Table)
features = Output("Features", AttributeList, dynamic=False)
settings_version = 2
settingsHandler = DomainContextHandler()
auto_send_selection = Setting(True)
auto_sample = Setting(True)
toolbar_selection = Setting(0)
attr_x = ContextSetting(None)
attr_y = ContextSetting(None)
#: Serialized selection state to be restored
selection_group = Setting(None, schema_only=True)
graph = SettingProvider(OWScatterPlotGraph)
jitter_sizes = [0, 0.1, 0.5, 1, 2, 3, 4, 5, 7, 10]
graph_name = "graph.plot_widget.plotItem"
class Information(OWWidget.Information):
sampled_sql = Msg("Large SQL table; showing a sample.")
def __init__(self):
super().__init__()
box = gui.vBox(self.mainArea, True, margin=0)
self.graph = OWScatterPlotGraph(self, box, "ScatterPlot")
box.layout().addWidget(self.graph.plot_widget)
plot = self.graph.plot_widget
axispen = QPen(self.palette().color(QPalette.Text))
axis = plot.getAxis("bottom")
axis.setPen(axispen)
axis = plot.getAxis("left")
axis.setPen(axispen)
self.data = None # Orange.data.Table
self.subset_data = None # Orange.data.Table
self.sql_data = None # Orange.data.sql.table.SqlTable
self.attribute_selection_list = None # list of Orange.data.Variable
self.__timer = QTimer(self, interval=1200)
self.__timer.timeout.connect(self.add_data)
#: Remember the saved state to restore
self.__pending_selection_restore = self.selection_group
self.selection_group = None
common_options = dict(
labelWidth=50, orientation=Qt.Horizontal, sendSelectedValue=True,
valueType=str)
box = gui.vBox(self.controlArea, "Axis Data")
dmod = DomainModel
self.xy_model = DomainModel(dmod.MIXED, valid_types=dmod.PRIMITIVE)
self.cb_attr_x = gui.comboBox(
box, self, "attr_x", label="Axis x:", callback=self.update_attr,
model=self.xy_model, **common_options)
self.cb_attr_y = gui.comboBox(
box, self, "attr_y", label="Axis y:", callback=self.update_attr,
model=self.xy_model, **common_options)
vizrank_box = gui.hBox(box)
gui.separator(vizrank_box, width=common_options["labelWidth"])
self.vizrank, self.vizrank_button = ScatterPlotVizRank.add_vizrank(
vizrank_box, self, "Find Informative Projections", self.set_attr)
gui.separator(box)
g = self.graph.gui
g.add_widgets([g.JitterSizeSlider,
g.JitterNumericValues], box)
self.sampling = gui.auto_commit(
self.controlArea, self, "auto_sample", "Sample", box="Sampling",
callback=self.switch_sampling, commit=lambda: self.add_data(1))
self.sampling.setVisible(False)
g.point_properties_box(self.controlArea)
self.models = [self.xy_model] + g.points_models
box_plot_prop = gui.vBox(self.controlArea, "Plot Properties")
g.add_widgets([g.ShowLegend,
g.ShowGridLines,
g.ToolTipShowsAll,
g.ClassDensity,
g.RegressionLine,
g.LabelOnlySelected], box_plot_prop)
self.graph.box_zoom_select(self.controlArea)
self.controlArea.layout().addStretch(100)
self.icons = gui.attributeIconDict
p = self.graph.plot_widget.palette()
self.graph.set_palette(p)
gui.auto_commit(self.controlArea, self, "auto_send_selection",
"Send Selection", "Send Automatically")
self.graph.zoom_actions(self)
def keyPressEvent(self, event):
super().keyPressEvent(event)
self.graph.update_tooltip(event.modifiers())
def keyReleaseEvent(self, event):
super().keyReleaseEvent(event)
self.graph.update_tooltip(event.modifiers())
def reset_graph_data(self, *_):
if self.data is not None:
self.graph.rescale_data()
self.update_graph()
def _vizrank_color_change(self):
self.vizrank.initialize()
is_enabled = self.data is not None and not self.data.is_sparse() and \
len([v for v in chain(self.data.domain.variables, self.data.domain.metas)
if v.is_primitive]) > 2\
and len(self.data) > 1
self.vizrank_button.setEnabled(
is_enabled and self.graph.attr_color is not None and
not np.isnan(self.data.get_column_view(self.graph.attr_color)[0].astype(float)).all())
if is_enabled and self.graph.attr_color is None:
self.vizrank_button.setToolTip("Color variable has to be selected.")
else:
self.vizrank_button.setToolTip("")
@Inputs.data
def set_data(self, data):
self.clear_messages()
self.Information.sampled_sql.clear()
self.__timer.stop()
self.sampling.setVisible(False)
self.sql_data = None
if isinstance(data, SqlTable):
if data.approx_len() < 4000:
data = Table(data)
else:
self.Information.sampled_sql()
self.sql_data = data
data_sample = data.sample_time(0.8, no_cache=True)
data_sample.download_data(2000, partial=True)
data = Table(data_sample)
self.sampling.setVisible(True)
if self.auto_sample:
self.__timer.start()
if data is not None and (len(data) == 0 or len(data.domain) == 0):
data = None
if self.data and data and self.data.checksum() == data.checksum():
return
self.closeContext()
same_domain = (self.data and data and
data.domain.checksum() == self.data.domain.checksum())
self.data = data
if not same_domain:
self.init_attr_values()
self.openContext(self.data)
self._vizrank_color_change()
def findvar(name, iterable):
"""Find a Orange.data.Variable in `iterable` by name"""
for el in iterable:
if isinstance(el, Orange.data.Variable) and el.name == name:
return el
return None
# handle restored settings from < 3.3.9 when attr_* were stored
# by name
if isinstance(self.attr_x, str):
self.attr_x = findvar(self.attr_x, self.xy_model)
if isinstance(self.attr_y, str):
self.attr_y = findvar(self.attr_y, self.xy_model)
if isinstance(self.graph.attr_label, str):
self.graph.attr_label = findvar(
self.graph.attr_label, self.graph.gui.label_model)
if isinstance(self.graph.attr_color, str):
self.graph.attr_color = findvar(
self.graph.attr_color, self.graph.gui.color_model)
if isinstance(self.graph.attr_shape, str):
self.graph.attr_shape = findvar(
self.graph.attr_shape, self.graph.gui.shape_model)
if isinstance(self.graph.attr_size, str):
self.graph.attr_size = findvar(
self.graph.attr_size, self.graph.gui.size_model)
def add_data(self, time=0.4):
if self.data and len(self.data) > 2000:
return self.__timer.stop()
data_sample = self.sql_data.sample_time(time, no_cache=True)
if data_sample:
data_sample.download_data(2000, partial=True)
data = Table(data_sample)
self.data = Table.concatenate((self.data, data), axis=0)
self.handleNewSignals()
def switch_sampling(self):
self.__timer.stop()
if self.auto_sample and self.sql_data:
self.add_data()
self.__timer.start()
@Inputs.data_subset
def set_subset_data(self, subset_data):
self.warning()
if isinstance(subset_data, SqlTable):
if subset_data.approx_len() < AUTO_DL_LIMIT:
subset_data = Table(subset_data)
else:
self.warning("Data subset does not support large Sql tables")
subset_data = None
self.subset_data = subset_data
self.controls.graph.alpha_value.setEnabled(subset_data is None)
# called when all signals are received, so the graph is updated only once
def handleNewSignals(self):
self.graph.new_data(self.data, self.subset_data)
if self.attribute_selection_list and self.graph.domain and \
all(attr in self.graph.domain
for attr in self.attribute_selection_list):
self.attr_x = self.attribute_selection_list[0]
self.attr_y = self.attribute_selection_list[1]
self.attribute_selection_list = None
self.update_graph()
self.cb_class_density.setEnabled(self.graph.can_draw_density())
self.cb_reg_line.setEnabled(self.graph.can_draw_regresssion_line())
if self.data is not None and self.__pending_selection_restore is not None:
self.apply_selection(self.__pending_selection_restore)
self.__pending_selection_restore = None
self.unconditional_commit()
def apply_selection(self, selection):
"""Apply `selection` to the current plot."""
if self.data is not None:
self.graph.selection = np.zeros(len(self.data), dtype=np.uint8)
self.selection_group = [x for x in selection if x[0] < len(self.data)]
selection_array = np.array(self.selection_group).T
self.graph.selection[selection_array[0]] = selection_array[1]
self.graph.update_colors(keep_colors=True)
@Inputs.features
def set_shown_attributes(self, attributes):
if attributes and len(attributes) >= 2:
self.attribute_selection_list = attributes[:2]
else:
self.attribute_selection_list = None
def init_attr_values(self):
domain = self.data and self.data.domain
for model in self.models:
model.set_domain(domain)
self.attr_x = self.xy_model[0] if self.xy_model else None
self.attr_y = self.xy_model[1] if len(self.xy_model) >= 2 \
else self.attr_x
self.graph.attr_color = self.data.domain.class_var if domain else None
self.graph.attr_shape = None
self.graph.attr_size = None
self.graph.attr_label = None
def set_attr(self, attr_x, attr_y):
self.attr_x, self.attr_y = attr_x, attr_y
self.update_attr()
def update_attr(self):
self.update_graph()
self.cb_class_density.setEnabled(self.graph.can_draw_density())
self.cb_reg_line.setEnabled(self.graph.can_draw_regresssion_line())
self.send_features()
def update_colors(self):
self._vizrank_color_change()
self.cb_class_density.setEnabled(self.graph.can_draw_density())
def update_density(self):
self.update_graph(reset_view=False)
def update_regression_line(self):
self.update_graph(reset_view=False)
def update_graph(self, reset_view=True, **_):
self.graph.zoomStack = []
if self.graph.data is None:
return
self.graph.update_data(self.attr_x, self.attr_y, reset_view)
def selection_changed(self):
# Store current selection in a setting that is stored in workflow
if isinstance(self.data, SqlTable):
selection = None
elif self.data is not None:
selection = self.graph.get_selection()
else:
selection = None
if selection is not None and len(selection):
self.selection_group = list(zip(selection, self.graph.selection[selection]))
else:
self.selection_group = None
self.commit()
def send_data(self):
# TODO: Implement selection for sql data
def _get_selected():
if not len(selection):
return None
return create_groups_table(data, graph.selection, False, "Group")
def _get_annotated():
if graph.selection is not None and np.max(graph.selection) > 1:
return create_groups_table(data, graph.selection)
else:
return create_annotated_table(data, selection)
graph = self.graph
data = self.data
selection = graph.get_selection()
self.Outputs.annotated_data.send(_get_annotated())
self.Outputs.selected_data.send(_get_selected())
def send_features(self):
features = [attr for attr in [self.attr_x, self.attr_y] if attr]
self.Outputs.features.send(features or None)
def commit(self):
self.send_data()
self.send_features()
def get_widget_name_extension(self):
if self.data is not None:
return "{} vs {}".format(self.attr_x.name, self.attr_y.name)
def send_report(self):
if self.data is None:
return
def name(var):
return var and var.name
caption = report.render_items_vert((
("Color", name(self.graph.attr_color)),
("Label", name(self.graph.attr_label)),
("Shape", name(self.graph.attr_shape)),
("Size", name(self.graph.attr_size)),
("Jittering", (self.attr_x.is_discrete or
self.attr_y.is_discrete or
self.graph.jitter_continuous) and
self.graph.jitter_size)))
self.report_plot()
if caption:
self.report_caption(caption)
def onDeleteWidget(self):
super().onDeleteWidget()
self.graph.plot_widget.getViewBox().deleteLater()
self.graph.plot_widget.clear()
@classmethod
def migrate_settings(cls, settings, version):
if version < 2 and "selection" in settings and settings["selection"]:
settings["selection_group"] = [(a, 1) for a in settings["selection"]]
class OWPCA(widget.OWWidget):
name = "PCA"
description = "Principal component analysis with a scree-diagram."
icon = "icons/PCA.svg"
priority = 3050
keywords = ["principal component analysis", "linear transformation"]
class Inputs:
data = Input("Data", Table)
class Outputs:
transformed_data = Output("Transformed data", Table)
components = Output("Components", Table)
pca = Output("PCA", PCA, dynamic=False)
preprocessor = Output("Preprocessor", Preprocess)
settingsHandler = settings.DomainContextHandler()
ncomponents = settings.Setting(2)
variance_covered = settings.Setting(100)
batch_size = settings.Setting(100)
address = settings.Setting('')
auto_update = settings.Setting(True)
auto_commit = settings.Setting(True)
normalize = settings.ContextSetting(True)
decomposition_idx = settings.ContextSetting(0)
maxp = settings.Setting(20)
axis_labels = settings.Setting(10)
graph_name = "plot.plotItem"
class Warning(widget.OWWidget.Warning):
trivial_components = widget.Msg(
"All components of the PCA are trivial (explain 0 variance). "
"Input data is constant (or near constant).")
class Error(widget.OWWidget.Error):
no_features = widget.Msg("At least 1 feature is required")
no_instances = widget.Msg("At least 1 data instance is required")
sparse_data = widget.Msg("Sparse data is not supported")
def __init__(self):
super().__init__()
self.data = None
self._pca = None
self._transformed = None
self._variance_ratio = None
self._cumulative = None
self._line = False
self._init_projector()
# Components Selection
box = gui.vBox(self.controlArea, "Components Selection")
form = QFormLayout()
box.layout().addLayout(form)
self.components_spin = gui.spin(
box, self, "ncomponents", 1, MAX_COMPONENTS,
callback=self._update_selection_component_spin,
keyboardTracking=False
)
self.components_spin.setSpecialValueText("All")
self.variance_spin = gui.spin(
box, self, "variance_covered", 1, 100,
callback=self._update_selection_variance_spin,
keyboardTracking=False
)
self.variance_spin.setSuffix("%")
form.addRow("Components:", self.components_spin)
form.addRow("Variance covered:", self.variance_spin)
# Incremental learning
self.sampling_box = gui.vBox(self.controlArea, "Incremental learning")
self.addresstext = QLineEdit(box)
self.addresstext.setPlaceholderText('Remote server')
if self.address:
self.addresstext.setText(self.address)
self.sampling_box.layout().addWidget(self.addresstext)
form = QFormLayout()
self.sampling_box.layout().addLayout(form)
self.batch_spin = gui.spin(
self.sampling_box, self, "batch_size", 50, 100000, step=50,
keyboardTracking=False)
form.addRow("Batch size ~ ", self.batch_spin)
self.start_button = gui.button(
self.sampling_box, self, "Start remote computation",
callback=self.start, autoDefault=False,
tooltip="Start/abort computation on the server")
self.start_button.setEnabled(False)
gui.checkBox(self.sampling_box, self, "auto_update",
"Periodically fetch model", callback=self.update_model)
self.__timer = QTimer(self, interval=2000)
self.__timer.timeout.connect(self.get_model)
self.sampling_box.setVisible(remotely)
# Decomposition
self.decomposition_box = gui.radioButtons(
self.controlArea, self,
"decomposition_idx", [d.name for d in DECOMPOSITIONS],
box="Decomposition", callback=self._update_decomposition
)
# Options
self.options_box = gui.vBox(self.controlArea, "Options")
self.normalize_box = gui.checkBox(
self.options_box, self, "normalize",
"Normalize data", callback=self._update_normalize
)
self.maxp_spin = gui.spin(
self.options_box, self, "maxp", 1, MAX_COMPONENTS,
label="Show only first", callback=self._setup_plot,
keyboardTracking=False
)
self.controlArea.layout().addStretch()
gui.auto_commit(self.controlArea, self, "auto_commit", "Apply",
checkbox_label="Apply automatically")
self.plot = pg.PlotWidget(background="w")
axis = self.plot.getAxis("bottom")
axis.setLabel("Principal Components")
axis = self.plot.getAxis("left")
axis.setLabel("Proportion of variance")
self.plot_horlabels = []
self.plot_horlines = []
self.plot.getViewBox().setMenuEnabled(False)
self.plot.getViewBox().setMouseEnabled(False, False)
self.plot.showGrid(True, True, alpha=0.5)
self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0))
self.mainArea.layout().addWidget(self.plot)
self._update_normalize()
def update_model(self):
self.get_model()
if self.auto_update and self.rpca and not self.rpca.ready():
self.__timer.start(2000)
else:
self.__timer.stop()
def update_buttons(self, sparse_data=False):
if sparse_data:
self.normalize = False
buttons = self.decomposition_box.buttons
for cls, button in zip(DECOMPOSITIONS, buttons):
button.setDisabled(sparse_data and not cls.supports_sparse)
if not buttons[self.decomposition_idx].isEnabled():
# Set decomposition index to first sparse-enabled decomposition
for i, cls in enumerate(DECOMPOSITIONS):
if cls.supports_sparse:
self.decomposition_idx = i
break
self._init_projector()
def start(self):
if 'Abort' in self.start_button.text():
self.rpca.abort()
self.__timer.stop()
self.start_button.setText("Start remote computation")
else:
self.address = self.addresstext.text()
with remote.server(self.address):
from Orange.projection.pca import RemotePCA
maxiter = (1e5 + self.data.approx_len()) / self.batch_size * 3
self.rpca = RemotePCA(self.data, self.batch_size, int(maxiter))
self.update_model()
self.start_button.setText("Abort remote computation")
@Inputs.data
def set_data(self, data):
self.closeContext()
self.clear_messages()
self.clear()
self.start_button.setEnabled(False)
self.information()
self.data = None
if isinstance(data, SqlTable):
if data.approx_len() < AUTO_DL_LIMIT:
data = Table(data)
elif not remotely:
self.information("Data has been sampled")
data_sample = data.sample_time(1, no_cache=True)
data_sample.download_data(2000, partial=True)
data = Table(data_sample)
else: # data was big and remote available
self.sampling_box.setVisible(True)
self.start_button.setText("Start remote computation")
self.start_button.setEnabled(True)
if not isinstance(data, SqlTable):
self.sampling_box.setVisible(False)
if isinstance(data, Table):
if len(data.domain.attributes) == 0:
self.Error.no_features()
self.clear_outputs()
return
if len(data) == 0:
self.Error.no_instances()
self.clear_outputs()
return
self.openContext(data)
sparse_data = data is not None and data.is_sparse()
self.normalize_box.setDisabled(sparse_data)
self.update_buttons(sparse_data=sparse_data)
self.data = data
self.fit()
def fit(self):
self.clear()
self.Warning.trivial_components.clear()
if self.data is None:
return
data = self.data
self._pca_projector.preprocessors = \
self._pca_preprocessors + ([Normalize()] if self.normalize else [])
if not isinstance(data, SqlTable):
pca = self._pca_projector(data)
variance_ratio = pca.explained_variance_ratio_
cumulative = numpy.cumsum(variance_ratio)
if numpy.isfinite(cumulative[-1]):
self.components_spin.setRange(0, len(cumulative))
self._pca = pca
self._variance_ratio = variance_ratio
self._cumulative = cumulative
self._setup_plot()
else:
self.Warning.trivial_components()
self.unconditional_commit()
def clear(self):
self._pca = None
self._transformed = None
self._variance_ratio = None
self._cumulative = None
self._line = None
self.plot_horlabels = []
self.plot_horlines = []
self.plot.clear()
def clear_outputs(self):
self.Outputs.transformed_data.send(None)
self.Outputs.components.send(None)
self.Outputs.pca.send(self._pca_projector)
self.Outputs.preprocessor.send(None)
def get_model(self):
if self.rpca is None:
return
if self.rpca.ready():
self.__timer.stop()
self.start_button.setText("Restart (finished)")
self._pca = self.rpca.get_state()
if self._pca is None:
return
self._variance_ratio = self._pca.explained_variance_ratio_
self._cumulative = numpy.cumsum(self._variance_ratio)
self._setup_plot()
self._transformed = None
self.commit()
def _setup_plot(self):
self.plot.clear()
if self._pca is None:
return
explained_ratio = self._variance_ratio
explained = self._cumulative
p = min(len(self._variance_ratio), self.maxp)
self.plot.plot(numpy.arange(p), explained_ratio[:p],
pen=pg.mkPen(QColor(Qt.red), width=2),
antialias=True,
name="Variance")
self.plot.plot(numpy.arange(p), explained[:p],
pen=pg.mkPen(QColor(Qt.darkYellow), width=2),
antialias=True,
name="Cumulative Variance")
cutpos = self._nselected_components() - 1
self._line = pg.InfiniteLine(
angle=90, pos=cutpos, movable=True, bounds=(0, p - 1))
self._line.setCursor(Qt.SizeHorCursor)
self._line.setPen(pg.mkPen(QColor(Qt.black), width=2))
self._line.sigPositionChanged.connect(self._on_cut_changed)
self.plot.addItem(self._line)
self.plot_horlines = (
pg.PlotCurveItem(pen=pg.mkPen(QColor(Qt.blue), style=Qt.DashLine)),
pg.PlotCurveItem(pen=pg.mkPen(QColor(Qt.blue), style=Qt.DashLine)))
self.plot_horlabels = (
pg.TextItem(color=QColor(Qt.black), anchor=(1, 0)),
pg.TextItem(color=QColor(Qt.black), anchor=(1, 1)))
for item in self.plot_horlabels + self.plot_horlines:
self.plot.addItem(item)
self._set_horline_pos()
self.plot.setRange(xRange=(0.0, p - 1), yRange=(0.0, 1.0))
self._update_axis()
def _set_horline_pos(self):
cutidx = self.ncomponents - 1
for line, label, curve in zip(self.plot_horlines, self.plot_horlabels,
(self._variance_ratio, self._cumulative)):
y = curve[cutidx]
line.setData([-1, cutidx], 2 * [y])
label.setPos(cutidx, y)
label.setPlainText("{:.3f}".format(y))
def _on_cut_changed(self, line):
# cut changed by means of a cut line over the scree plot.
value = int(round(line.value()))
self._line.setValue(value)
current = self._nselected_components()
components = value + 1
if not (self.ncomponents == 0 and
components == len(self._variance_ratio)):
self.ncomponents = components
self._set_horline_pos()
if self._pca is not None:
var = self._cumulative[components - 1]
if numpy.isfinite(var):
self.variance_covered = int(var * 100)
if current != self._nselected_components():
self._invalidate_selection()
def _update_selection_component_spin(self):
# cut changed by "ncomponents" spin.
if self._pca is None:
self._invalidate_selection()
return
if self.ncomponents == 0:
# Special "All" value
cut = len(self._variance_ratio)
else:
cut = self.ncomponents
var = self._cumulative[cut - 1]
if numpy.isfinite(var):
self.variance_covered = int(var * 100)
if numpy.floor(self._line.value()) + 1 != cut:
self._line.setValue(cut - 1)
self._invalidate_selection()
def _update_selection_variance_spin(self):
# cut changed by "max variance" spin.
if self._pca is None:
return
cut = numpy.searchsorted(self._cumulative,
self.variance_covered / 100.0) + 1
cut = min(cut, len(self._cumulative))
self.ncomponents = cut
if numpy.floor(self._line.value()) + 1 != cut:
self._line.setValue(cut - 1)
self._invalidate_selection()
def _update_normalize(self):
self.fit()
if self.data is None:
self._invalidate_selection()
def _init_projector(self):
cls = DECOMPOSITIONS[self.decomposition_idx]
self._pca_projector = cls(n_components=MAX_COMPONENTS)
self._pca_projector.component = self.ncomponents
self._pca_preprocessors = cls.preprocessors
def _update_decomposition(self):
self._init_projector()
self._update_normalize()
def _nselected_components(self):
"""Return the number of selected components."""
if self._pca is None:
return 0
if self.ncomponents == 0:
# Special "All" value
max_comp = len(self._variance_ratio)
else:
max_comp = self.ncomponents
var_max = self._cumulative[max_comp - 1]
if var_max != numpy.floor(self.variance_covered / 100.0):
cut = max_comp
assert numpy.isfinite(var_max)
self.variance_covered = int(var_max * 100)
else:
self.ncomponents = cut = numpy.searchsorted(
self._cumulative, self.variance_covered / 100.0) + 1
return cut
def _invalidate_selection(self):
self.commit()
def _update_axis(self):
p = min(len(self._variance_ratio), self.maxp)
axis = self.plot.getAxis("bottom")
d = max((p-1)//(self.axis_labels-1), 1)
axis.setTicks([[(i, str(i+1)) for i in range(0, p, d)]])
def commit(self):
transformed = components = pp = None
if self._pca is not None:
if self._transformed is None:
# Compute the full transform (MAX_COMPONENTS components) only once.
self._transformed = self._pca(self.data)
transformed = self._transformed
domain = Domain(
transformed.domain.attributes[:self.ncomponents],
self.data.domain.class_vars,
self.data.domain.metas
)
transformed = transformed.from_table(domain, transformed)
# prevent caching new features by defining compute_value
dom = Domain([ContinuousVariable(a.name, compute_value=lambda _: None)
for a in self._pca.orig_domain.attributes],
metas=[StringVariable(name='component')])
metas = numpy.array([['PC{}'.format(i + 1)
for i in range(self.ncomponents)]],
dtype=object).T
components = Table(dom, self._pca.components_[:self.ncomponents],
metas=metas)
components.name = 'components'
pp = ApplyDomain(domain, "PCA")
self._pca_projector.component = self.ncomponents
self.Outputs.transformed_data.send(transformed)
self.Outputs.components.send(components)
self.Outputs.pca.send(self._pca_projector)
self.Outputs.preprocessor.send(pp)
def send_report(self):
if self.data is None:
return
self.report_items((
("Decomposition", DECOMPOSITIONS[self.decomposition_idx].name),
("Normalize data", str(self.normalize)),
("Selected components", self.ncomponents),
("Explained variance", "{:.3f} %".format(self.variance_covered))
))
self.report_plot()
@classmethod
def migrate_settings(cls, settings, version):
if "variance_covered" in settings:
# Due to the error in gh-1896 the variance_covered was persisted
# as a NaN value, causing a TypeError in the widgets `__init__`.
vc = settings["variance_covered"]
if isinstance(vc, numbers.Real):
if numpy.isfinite(vc):
vc = int(vc)
else:
vc = 100
settings["variance_covered"] = vc
if settings.get("ncomponents", 0) > MAX_COMPONENTS:
settings["ncomponents"] = MAX_COMPONENTS
class EventSpy(QObject):
"""
A testing utility class (similar to QSignalSpy) to record events
delivered to a QObject instance.
Note
----
Only event types can be recorded (as QEvent instances are deleted
on delivery).
Note
----
Can only be used with a QCoreApplication running.
Parameters
----------
object : QObject
An object whose events need to be recorded.
etype : Union[QEvent.Type, Sequence[QEvent.Type]
A event type (or types) that should be recorded
"""
def __init__(self, object, etype, **kwargs):
super().__init__(**kwargs)
if not isinstance(object, QObject):
raise TypeError
self.__object = object
try:
len(etype)
except TypeError:
etypes = {etype}
else:
etypes = set(etype)
self.__etypes = etypes
self.__record = []
self.__loop = QEventLoop()
self.__timer = QTimer(self, singleShot=True)
self.__timer.timeout.connect(self.__loop.quit)
self.__object.installEventFilter(self)
def wait(self, timeout=5000):
"""
Start an event loop that runs until a spied event or a timeout occurred.
Parameters
----------
timeout : int
Timeout in milliseconds.
Returns
-------
res : bool
True if the event occurred and False otherwise.
Example
-------
>>> app = QCoreApplication.instance() or QCoreApplication([])
>>> obj = QObject()
>>> spy = EventSpy(obj, QEvent.User)
>>> app.postEvent(obj, QEvent(QEvent.User))
>>> spy.wait()
True
>>> print(spy.events())
[1000]
"""
count = len(self.__record)
self.__timer.stop()
self.__timer.setInterval(timeout)
self.__timer.start()
self.__loop.exec_()
self.__timer.stop()
return len(self.__record) != count
def eventFilter(self, reciever, event):
if reciever is self.__object and event.type() in self.__etypes:
self.__record.append(event.type())
if self.__loop.isRunning():
self.__loop.quit()
return super().eventFilter(reciever, event)
def events(self):
"""
Return a list of all (listened to) event types that occurred.
Returns
-------
events : List[QEvent.Type]
"""
return list(self.__record)
class OWLouvainClustering(widget.OWWidget):
name = 'Louvain Clustering'
description = 'Detects communities in a network of nearest neighbors.'
icon = 'icons/LouvainClustering.svg'
priority = 2110
want_main_area = False
settingsHandler = DomainContextHandler()
class Inputs:
data = Input('Data', Table, default=True)
if Graph is not None:
class Outputs:
annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME, Table, default=True)
graph = Output('Network', Graph)
else:
class Outputs:
annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME, Table, default=True)
apply_pca = ContextSetting(True)
pca_components = ContextSetting(_DEFAULT_PCA_COMPONENTS)
metric_idx = ContextSetting(0)
k_neighbors = ContextSetting(_DEFAULT_K_NEIGHBORS)
resolution = ContextSetting(1.)
auto_commit = Setting(False)
class Information(widget.OWWidget.Information):
modified = Msg("Press commit to recompute clusters and send new data")
class Error(widget.OWWidget.Error):
empty_dataset = Msg('No features in data')
general_error = Msg('Error occured during clustering\n{}')
def __init__(self):
super().__init__()
self.data = None # type: Optional[Table]
self.preprocessed_data = None # type: Optional[Table]
self.pca_projection = None # type: Optional[Table]
self.graph = None # type: Optional[nx.Graph]
self.partition = None # type: Optional[np.array]
# Use a executor with a single worker, to limit CPU overcommitment for
# cancelled tasks. The method does not have a fine cancellation
# granularity so we assure that there are not N - 1 jobs executing
# for no reason only to be thrown away. It would be better to use the
# global pool but implement a limit on jobs from this source.
self.__executor = futures.ThreadPoolExecutor(max_workers=1)
self.__task = None # type: Optional[TaskState]
self.__invalidated = False
# coalescing commit timer
self.__commit_timer = QTimer(self, singleShot=True)
self.__commit_timer.timeout.connect(self.commit)
pca_box = gui.vBox(self.controlArea, 'PCA Preprocessing')
self.apply_pca_cbx = gui.checkBox(
pca_box, self, 'apply_pca', label='Apply PCA preprocessing',
callback=self._invalidate_graph,
) # type: QCheckBox
self.pca_components_slider = gui.hSlider(
pca_box, self, 'pca_components', label='Components: ', minValue=2,
maxValue=_MAX_PCA_COMPONENTS,
callback=self._invalidate_pca_projection, tracking=False
) # type: QSlider
graph_box = gui.vBox(self.controlArea, 'Graph parameters')
self.metric_combo = gui.comboBox(
graph_box, self, 'metric_idx', label='Distance metric',
items=[m[0] for m in METRICS], callback=self._invalidate_graph,
orientation=Qt.Horizontal,
) # type: gui.OrangeComboBox
self.k_neighbors_spin = gui.spin(
graph_box, self, 'k_neighbors', minv=1, maxv=_MAX_K_NEIGBOURS,
label='k neighbors', controlWidth=80, alignment=Qt.AlignRight,
callback=self._invalidate_graph,
) # type: gui.SpinBoxWFocusOut
self.resolution_spin = gui.hSlider(
graph_box, self, 'resolution', minValue=0, maxValue=5., step=1e-1,
label='Resolution', intOnly=False, labelFormat='%.1f',
callback=self._invalidate_partition, tracking=False,
) # type: QSlider
self.resolution_spin.parent().setToolTip(
'The resolution parameter affects the number of clusters to find. '
'Smaller values tend to produce more clusters and larger values '
'retrieve less clusters.'
)
self.apply_button = gui.auto_commit(
self.controlArea, self, 'auto_commit', 'Apply', box=None,
commit=lambda: self.commit(),
callback=lambda: self._on_auto_commit_changed(),
) # type: QWidget
def _invalidate_pca_projection(self):
self.pca_projection = None
if not self.apply_pca:
return
self._invalidate_graph()
self._set_modified(True)
def _invalidate_graph(self):
self.graph = None
self._invalidate_partition()
self._set_modified(True)
def _invalidate_partition(self):
self.partition = None
self._invalidate_output()
self.Information.modified()
self._set_modified(True)
def _invalidate_output(self):
self.__invalidated = True
if self.__task is not None:
self.__cancel_task(wait=False)
if self.auto_commit:
self.__commit_timer.start()
else:
self.__set_state_ready()
def _set_modified(self, state):
"""
Mark the widget (GUI) as containing modified state.
"""
if self.data is None:
# does not apply when we have no data
state = False
elif self.auto_commit:
# does not apply when auto commit is on
state = False
self.Information.modified(shown=state)
def _on_auto_commit_changed(self):
if self.auto_commit and self.__invalidated:
self.commit()
def cancel(self):
"""Cancel any running jobs."""
self.__cancel_task(wait=False)
self.__set_state_ready()
def commit(self):
self.__commit_timer.stop()
self.__invalidated = False
self._set_modified(False)
self.Error.clear()
# Cancel current running task
self.__cancel_task(wait=False)
if self.data is None:
self.__set_state_ready()
return
# Make sure the dataset is ok
if len(self.data.domain.attributes) < 1:
self.Error.empty_dataset()
self.__set_state_ready()
return
if self.partition is not None:
self.__set_state_ready()
self._send_data()
return
# Preprocess the dataset
if self.preprocessed_data is None:
louvain = Louvain(random_state=0)
self.preprocessed_data = louvain.preprocess(self.data)
state = TaskState(self)
# Prepare/assemble the task(s) to run; reuse partial results
if self.apply_pca:
if self.pca_projection is not None:
data = self.pca_projection
pca_components = None
else:
data = self.preprocessed_data
pca_components = self.pca_components
else:
data = self.preprocessed_data
pca_components = None
if self.graph is not None:
# run on graph only; no need to do PCA and k-nn search ...
graph = self.graph
k_neighbors = metric = None
else:
k_neighbors, metric = self.k_neighbors, METRICS[self.metric_idx][1]
graph = None
if graph is None:
task = partial(
run_on_data, data, pca_components=pca_components,
k_neighbors=k_neighbors, metric=metric,
resolution=self.resolution, state=state
)
else:
task = partial(
run_on_graph, graph, resolution=self.resolution, state=state
)
self.__set_state_busy()
self.__start_task(task, state)
@Slot(object)
def __set_partial_results(self, result):
# type: (Tuple[str, Any]) -> None
which, res = result
if which == "pca_projection":
assert isinstance(res, Table) and len(res) == len(self.data)
self.pca_projection = res
elif which == "graph":
assert isinstance(res, nx.Graph)
self.graph = res
elif which == "partition":
assert isinstance(res, np.ndarray)
self.partition = res
else:
assert False, which
@Slot(object)
def __on_done(self, future):
# type: (Future['Results']) -> None
assert future.done()
assert self.__task is not None
assert self.__task.future is future
assert self.__task.watcher.future() is future
self.__task, task = None, self.__task
task.deleteLater()
self.__set_state_ready()
try:
result = future.result()
except Exception as err: # pylint: disable=broad-except
self.Error.general_error(str(err), exc_info=True)
else:
self.__set_results(result)
@Slot(str)
def setStatusMessage(self, text):
super().setStatusMessage(text)
@Slot(float)
def progressBarSet(self, value, *a, **kw):
super().progressBarSet(value, *a, **kw)
def __set_state_ready(self):
self.progressBarFinished()
self.setBlocking(False)
self.setStatusMessage("")
def __set_state_busy(self):
self.progressBarInit()
self.setBlocking(True)
def __start_task(self, task, state):
# type: (Callable[[], Any], TaskState) -> None
assert self.__task is None
state.status_changed.connect(self.setStatusMessage)
state.progress_changed.connect(self.progressBarSet)
state.partial_result_ready.connect(self.__set_partial_results)
state.watcher.done.connect(self.__on_done)
state.start(self.__executor, task)
state.setParent(self)
self.__task = state
def __cancel_task(self, wait=True):
# Cancel and dispose of the current task
if self.__task is not None:
state, self.__task = self.__task, None
state.cancel()
state.partial_result_ready.disconnect(self.__set_partial_results)
state.status_changed.disconnect(self.setStatusMessage)
state.progress_changed.disconnect(self.progressBarSet)
state.watcher.done.disconnect(self.__on_done)
if wait:
futures.wait([state.future])
state.deleteLater()
else:
w = FutureWatcher(state.future, parent=state)
w.done.connect(state.deleteLater)
def __set_results(self, results):
# type: ('Results') -> None
# NOTE: All of these have already been set by __set_partial_results,
# we double check that they are aliases
if results.pca_projection is not None:
assert self.pca_components == results.pca_components
assert self.pca_projection is results.pca_projection
self.pca_projection = results.pca_projection
if results.graph is not None:
assert results.metric == METRICS[self.metric_idx][1]
assert results.k_neighbors == self.k_neighbors
assert self.graph is results.graph
self.graph = results.graph
if results.partition is not None:
assert results.resolution == self.resolution
assert self.partition is results.partition
self.partition = results.partition
self._send_data()
def _send_data(self):
if self.partition is None or self.data is None:
return
domain = self.data.domain
# Compute the frequency of each cluster index
counts = np.bincount(self.partition)
indices = np.argsort(counts)[::-1]
index_map = {n: o for n, o in zip(indices, range(len(indices)))}
new_partition = list(map(index_map.get, self.partition))
cluster_var = DiscreteVariable(
get_unique_names(domain, 'Cluster'),
values=['C%d' % (i + 1) for i, _ in enumerate(np.unique(new_partition))]
)
new_domain = add_columns(domain, metas=[cluster_var])
new_table = self.data.transform(new_domain)
new_table.get_column_view(cluster_var)[0][:] = new_partition
self.Outputs.annotated_data.send(new_table)
if Graph is not None:
graph = Graph(self.graph)
graph.set_items(new_table)
self.Outputs.graph.send(graph)
@Inputs.data
def set_data(self, data):
self.closeContext()
self.Error.clear()
prev_data, self.data = self.data, data
self.openContext(self.data)
# If X hasn't changed, there's no reason to recompute clusters
if prev_data and self.data and np.array_equal(self.data.X, prev_data.X):
if self.auto_commit:
self._send_data()
return
# Clear the outputs
self.Outputs.annotated_data.send(None)
if Graph is not None:
self.Outputs.graph.send(None)
# Clear internal state
self.clear()
self._invalidate_pca_projection()
if self.data is None:
return
# Can't have more PCA components than the number of attributes
n_attrs = len(data.domain.attributes)
self.pca_components_slider.setMaximum(min(_MAX_PCA_COMPONENTS, n_attrs))
self.pca_components_slider.setValue(min(_DEFAULT_PCA_COMPONENTS, n_attrs))
# Can't have more k neighbors than there are data points
self.k_neighbors_spin.setMaximum(min(_MAX_K_NEIGBOURS, len(data) - 1))
self.k_neighbors_spin.setValue(min(_DEFAULT_K_NEIGHBORS, len(data) - 1))
self.commit()
def clear(self):
self.__cancel_task(wait=False)
self.preprocessed_data = None
self.pca_projection = None
self.graph = None
self.partition = None
self.Error.clear()
self.Information.modified.clear()
def onDeleteWidget(self):
self.__cancel_task(wait=True)
self.__executor.shutdown(True)
self.clear()
self.data = None
super().onDeleteWidget()
def send_report(self):
pca = report.bool_str(self.apply_pca)
if self.apply_pca:
pca += report.plural(', {number} component{s}', self.pca_components)
self.report_items((
('PCA preprocessing', pca),
('Metric', METRICS[self.metric_idx][0]),
('k neighbors', self.k_neighbors),
('Resolution', self.resolution),
))
class OWTimeSlice(widget.OWWidget):
name = 'Time Slice'
description = 'Select a slice of measurements on a time interval.'
icon = 'icons/TimeSlice.svg'
priority = 550
class Inputs:
data = Input("Data", Table)
class Outputs:
subset = Output("Subset", Table)
want_main_area = False
class Error(widget.OWWidget.Error):
no_time_variable = widget.Msg('Data contains no time variable')
MAX_SLIDER_VALUE = 500
DATE_FORMATS = ('yyyy-MM-dd', 'HH:mm:ss.zzz')
OVERLAP_AMOUNTS = OrderedDict((
('all but one (= shift by one slider value)', 0),
('6/7 of interval', 6/7),
('3/4 of interval', 3/4),
('1/2 of interval', 1/2),
('1/3 of interval', 1/3),
('1/5 of interval', 1/5)))
loop_playback = settings.Setting(True)
steps_overlap = settings.Setting(True)
overlap_amount = settings.Setting(next(iter(OVERLAP_AMOUNTS)))
playback_interval = settings.Setting(1000)
slider_values = settings.Setting((0, .2 * MAX_SLIDER_VALUE))
def __init__(self):
super().__init__()
self._delta = 0
self.play_timer = QTimer(self,
interval=self.playback_interval,
timeout=self.play_single_step)
slider = self.slider = Slider(Qt.Horizontal, self,
minimum=0, maximum=self.MAX_SLIDER_VALUE,
tracking=False,
valuesChanged=self.valuesChanged,
minimumValue=self.slider_values[0],
maximumValue=self.slider_values[1],)
slider.setShowText(False)
box = gui.vBox(self.controlArea, 'Time Slice')
box.layout().addWidget(slider)
hbox = gui.hBox(box)
def _dateTimeChanged(editted):
def handler():
minTime = self.date_from.dateTime().toMSecsSinceEpoch() / 1000
maxTime = self.date_to.dateTime().toMSecsSinceEpoch() / 1000
if minTime > maxTime:
minTime = maxTime = minTime if editted == self.date_from else maxTime
other = self.date_to if editted == self.date_from else self.date_from
with blockSignals(other):
other.setDateTime(editted.dateTime())
with blockSignals(self.slider):
self.slider.setValues(self.slider.unscale(minTime),
self.slider.unscale(maxTime))
self.send_selection(minTime, maxTime)
return handler
kwargs = dict(calendarPopup=True,
displayFormat=' '.join(self.DATE_FORMATS),
timeSpec=Qt.UTC)
date_from = self.date_from = QDateTimeEdit(self, **kwargs)
date_to = self.date_to = QDateTimeEdit(self, **kwargs)
date_from.dateTimeChanged.connect(_dateTimeChanged(date_from))
date_to.dateTimeChanged.connect(_dateTimeChanged(date_to))
hbox.layout().addStretch(100)
hbox.layout().addWidget(date_from)
hbox.layout().addWidget(QLabel(' – '))
hbox.layout().addWidget(date_to)
hbox.layout().addStretch(100)
vbox = gui.vBox(self.controlArea, 'Step / Play Through')
gui.checkBox(vbox, self, 'loop_playback',
label='Loop playback')
hbox = gui.hBox(vbox)
gui.checkBox(hbox, self, 'steps_overlap',
label='Stepping overlaps by:',
toolTip='If enabled, the active interval moves forward '
'(backward) by half of the interval at each step.')
gui.comboBox(hbox, self, 'overlap_amount',
items=tuple(self.OVERLAP_AMOUNTS.keys()),
sendSelectedValue=True)
gui.spin(vbox, self, 'playback_interval',
label='Playback delay (msec):',
minv=100, maxv=30000, step=200,
callback=lambda: self.play_timer.setInterval(self.playback_interval))
hbox = gui.hBox(vbox)
self.step_backward = gui.button(hbox, self, '⏮',
callback=lambda: self.play_single_step(backward=True),
autoDefault=False)
self.play_button = gui.button(hbox, self, '▶',
callback=self.playthrough,
toggleButton=True, default=True)
self.step_forward = gui.button(hbox, self, '⏭',
callback=self.play_single_step,
autoDefault=False)
gui.rubber(self.controlArea)
self._set_disabled(True)
def valuesChanged(self, minValue, maxValue):
self.slider_values = (minValue, maxValue)
self._delta = max(1, (maxValue - minValue))
minTime = self.slider.scale(minValue)
maxTime = self.slider.scale(maxValue)
from_dt = QDateTime.fromMSecsSinceEpoch(minTime * 1000).toUTC()
to_dt = QDateTime.fromMSecsSinceEpoch(maxTime * 1000).toUTC()
with blockSignals(self.date_from,
self.date_to):
self.date_from.setDateTime(from_dt)
self.date_to.setDateTime(to_dt)
self.send_selection(minTime, maxTime)
def send_selection(self, minTime, maxTime):
try:
time_values = self.data.time_values
except AttributeError:
return
indices = (minTime <= time_values) & (time_values <= maxTime)
self.Outputs.subset.send(self.data[indices] if indices.any() else None)
def playthrough(self):
playing = self.play_button.isChecked()
for widget in (self.slider,
self.step_forward,
self.step_backward):
widget.setDisabled(playing)
if playing:
self.play_timer.start()
self.play_button.setText('▮▮')
else:
self.play_timer.stop()
self.play_button.setText('▶')
def play_single_step(self, backward=False):
op = operator.sub if backward else operator.add
minValue, maxValue = self.slider.values()
orig_delta = delta = self._delta
if self.steps_overlap:
overlap_amount = self.OVERLAP_AMOUNTS[self.overlap_amount]
if overlap_amount:
delta = max(1, int(round(delta * (1 - overlap_amount))))
else:
delta = 1 # single slider step (== 1/self.MAX_SLIDER_VALUE)
if maxValue == self.slider.maximum() and not backward:
minValue = self.slider.minimum()
maxValue = minValue + orig_delta
if not self.loop_playback:
self.play_button.click()
assert not self.play_timer.isActive()
assert not self.play_button.isChecked()
elif minValue == self.slider.minimum() and backward:
maxValue = self.slider.maximum()
minValue = maxValue - orig_delta
else:
minValue = op(minValue, delta)
maxValue = op(maxValue, delta)
# Blocking signals because we want this to be synchronous to avoid
# re-setting self._delta
with blockSignals(self.slider):
self.slider.setValues(minValue, maxValue)
self.valuesChanged(self.slider.minimumValue(), self.slider.maximumValue())
self._delta = orig_delta # Override valuesChanged handler
def _set_disabled(self, is_disabled):
for func in [self.date_from, self.date_to, self.step_backward, self.play_button,
self.step_forward, self.controls.loop_playback,
self.controls.steps_overlap, self.controls.overlap_amount,
self.controls.playback_interval, self.slider]:
func.setDisabled(is_disabled)
@Inputs.data
def set_data(self, data):
slider = self.slider
self.data = data = None if data is None else Timeseries.from_data_table(data)
def disabled():
slider.setFormatter(str)
slider.setHistogram(None)
slider.setScale(0, 0)
slider.setValues(0, 0)
self._set_disabled(True)
self.Outputs.subset.send(None)
if data is None:
disabled()
return
if not isinstance(data.time_variable, TimeVariable):
self.Error.no_time_variable()
disabled()
return
self.Error.clear()
var = data.time_variable
time_values = data.time_values
self._set_disabled(False)
slider.setHistogram(time_values)
slider.setFormatter(var.repr_val)
slider.setScale(time_values.min(), time_values.max())
self.valuesChanged(slider.minimumValue(), slider.maximumValue())
# Update datetime edit fields
min_dt = QDateTime.fromMSecsSinceEpoch(time_values[0] * 1000).toUTC()
max_dt = QDateTime.fromMSecsSinceEpoch(time_values[-1] * 1000).toUTC()
self.date_from.setDateTimeRange(min_dt, max_dt)
self.date_to.setDateTimeRange(min_dt, max_dt)
date_format = ' '.join((self.DATE_FORMATS[0] if var.have_date else '',
self.DATE_FORMATS[1] if var.have_time else '')).strip()
self.date_from.setDisplayFormat(date_format)
self.date_to.setDisplayFormat(date_format)
class EventSpy(QObject):
"""
A testing utility class (similar to QSignalSpy) to record events
delivered to a QObject instance.
Note
----
Only event types can be recorded (as QEvent instances are deleted
on delivery).
Note
----
Can only be used with a QCoreApplication running.
Parameters
----------
object : QObject
An object whose events need to be recorded.
etype : Union[QEvent.Type, Sequence[QEvent.Type]
A event type (or types) that should be recorded
"""
def __init__(self, object, etype, **kwargs):
super().__init__(**kwargs)
if not isinstance(object, QObject):
raise TypeError
self.__object = object
try:
len(etype)
except TypeError:
etypes = {etype}
else:
etypes = set(etype)
self.__etypes = etypes
self.__record = []
self.__loop = QEventLoop()
self.__timer = QTimer(self, singleShot=True)
self.__timer.timeout.connect(self.__loop.quit)
self.__object.installEventFilter(self)
def wait(self, timeout=5000):
"""
Start an event loop that runs until a spied event or a timeout occurred.
Parameters
----------
timeout : int
Timeout in milliseconds.
Returns
-------
res : bool
True if the event occurred and False otherwise.
Example
-------
>>> app = QCoreApplication.instance() or QCoreApplication([])
>>> obj = QObject()
>>> spy = EventSpy(obj, QEvent.User)
>>> app.postEvent(obj, QEvent(QEvent.User))
>>> spy.wait()
True
>>> print(spy.events())
[1000]
"""
count = len(self.__record)
self.__timer.stop()
self.__timer.setInterval(timeout)
self.__timer.start()
self.__loop.exec_()
self.__timer.stop()
return len(self.__record) != count
def eventFilter(self, reciever, event):
if reciever is self.__object and event.type() in self.__etypes:
self.__record.append(event.type())
if self.__loop.isRunning():
self.__loop.quit()
return super().eventFilter(reciever, event)
def events(self):
"""
Return a list of all (listened to) event types that occurred.
Returns
-------
events : List[QEvent.Type]
"""
return list(self.__record)
class VizRankDialog(QDialog, ProgressBarMixin, WidgetMessagesMixin):
"""
Base class for VizRank dialogs, providing a GUI with a table and a button,
and the skeleton for managing the evaluation of visualizations.
Derived classes must provide methods
- `iterate_states` for generating combinations (e.g. pairs of attritutes),
- `compute_score(state)` for computing the score of a combination,
- `row_for_state(state)` that returns a list of items inserted into the
table for the given state.
and, optionally,
- `state_count` that returns the number of combinations (used for progress
bar)
- `on_selection_changed` that handles event triggered when the user selects
a table row. The method should emit signal
`VizRankDialog.selectionChanged(object)`.
- `bar_length` returns the length of the bar corresponding to the score.
The class provides a table and a button. A widget constructs a single
instance of this dialog in its `__init__`, like (in Sieve) by using a
convenience method :obj:`add_vizrank`::
self.vizrank, self.vizrank_button = SieveRank.add_vizrank(
box, self, "Score Combinations", self.set_attr)
When the widget receives new data, it must call the VizRankDialog's
method :obj:`VizRankDialog.initialize()` to clear the GUI and reset the
state.
Clicking the Start button calls method `run` (and renames the button to
Pause). Run sets up a progress bar by getting the number of combinations
from :obj:`VizRankDialog.state_count()`. It restores the paused state
(if any) and calls generator :obj:`VizRankDialog.iterate_states()`. For
each generated state, it calls :obj:`VizRankDialog.score(state)`, which
must return the score (lower is better) for this state. If the returned
state is not `None`, the data returned by `row_for_state` is inserted at
the appropriate place in the table.
Args:
master (Orange.widget.OWWidget): widget to which the dialog belongs
Attributes:
master (Orange.widget.OWWidget): widget to which the dialog belongs
captionTitle (str): the caption for the dialog. This can be a class
attribute. `captionTitle` is used by the `ProgressBarMixin`.
"""
captionTitle = ""
processingStateChanged = Signal(int)
progressBarValueChanged = Signal(float)
messageActivated = Signal(Msg)
messageDeactivated = Signal(Msg)
selectionChanged = Signal(object)
class Information(WidgetMessagesMixin.Information):
nothing_to_rank = Msg("There is nothing to rank.")
def __init__(self, master):
"""Initialize the attributes and set up the interface"""
QDialog.__init__(self, master, windowTitle=self.captionTitle)
WidgetMessagesMixin.__init__(self)
self.setLayout(QVBoxLayout())
self.insert_message_bar()
self.layout().insertWidget(0, self.message_bar)
self.master = master
self.keep_running = False
self.scheduled_call = None
self.saved_state = None
self.saved_progress = 0
self.scores = []
self.add_to_model = queue.Queue()
self.update_timer = QTimer(self)
self.update_timer.timeout.connect(self._update)
self.update_timer.setInterval(200)
self._thread = None
self._worker = None
self.filter = QLineEdit()
self.filter.setPlaceholderText("Filter ...")
self.filter.textChanged.connect(self.filter_changed)
self.layout().addWidget(self.filter)
# Remove focus from line edit
self.setFocus(Qt.ActiveWindowFocusReason)
self.rank_model = QStandardItemModel(self)
self.model_proxy = QSortFilterProxyModel(
self, filterCaseSensitivity=False)
self.model_proxy.setSourceModel(self.rank_model)
self.rank_table = view = QTableView(
selectionBehavior=QTableView.SelectRows,
selectionMode=QTableView.SingleSelection,
showGrid=False,
editTriggers=gui.TableView.NoEditTriggers)
if self._has_bars:
view.setItemDelegate(TableBarItem())
else:
view.setItemDelegate(HorizontalGridDelegate())
view.setModel(self.model_proxy)
view.selectionModel().selectionChanged.connect(
self.on_selection_changed)
view.horizontalHeader().setStretchLastSection(True)
view.horizontalHeader().hide()
self.layout().addWidget(view)
self.button = gui.button(
self, self, "Start", callback=self.toggle, default=True)
@property
def _has_bars(self):
return type(self).bar_length is not VizRankDialog.bar_length
@classmethod
def add_vizrank(cls, widget, master, button_label, set_attr_callback):
"""
Equip the widget with VizRank button and dialog, and monkey patch the
widget's `closeEvent` and `hideEvent` to close/hide the vizrank, too.
Args:
widget (QWidget): the widget into whose layout to insert the button
master (Orange.widgets.widget.OWWidget): the master widget
button_label: the label for the button
set_attr_callback: the callback for setting the projection chosen
in the vizrank
Returns:
tuple with Vizrank dialog instance and push button
"""
# Monkey patching could be avoided by mixing-in the class (not
# necessarily a good idea since we can make a mess of multiple
# defined/derived closeEvent and hideEvent methods). Furthermore,
# per-class patching would be better than per-instance, but we don't
# want to mess with meta-classes either.
vizrank = cls(master)
button = gui.button(
widget, master, button_label, callback=vizrank.reshow,
enabled=False)
vizrank.selectionChanged.connect(lambda args: set_attr_callback(*args))
master_close_event = master.closeEvent
master_hide_event = master.hideEvent
master_delete_event = master.onDeleteWidget
def closeEvent(event):
vizrank.close()
master_close_event(event)
def hideEvent(event):
vizrank.hide()
master_hide_event(event)
def deleteEvent():
vizrank.keep_running = False
if vizrank._thread is not None and vizrank._thread.isRunning():
vizrank._thread.quit()
vizrank._thread.wait()
master_delete_event()
master.closeEvent = closeEvent
master.hideEvent = hideEvent
master.onDeleteWidget = deleteEvent
return vizrank, button
def reshow(self):
"""Put the widget on top of all windows
"""
self.show()
self.raise_()
self.activateWindow()
def initialize(self):
"""
Clear and initialize the dialog.
This method must be called by the widget when the data is reset,
e.g. from `set_data` handler.
"""
if self._thread is not None and self._thread.isRunning():
self.keep_running = False
self._thread.quit()
self._thread.wait()
self.keep_running = False
self.scheduled_call = None
self.saved_state = None
self.saved_progress = 0
self.update_timer.stop()
self.progressBarFinished()
self.scores = []
self._update_model() # empty queue
self.rank_model.clear()
self.button.setText("Start")
self.button.setEnabled(self.check_preconditions())
self._thread = QThread(self)
self._worker = Worker(self)
self._worker.moveToThread(self._thread)
self._worker.stopped.connect(self._thread.quit)
self._worker.stopped.connect(self._select_first_if_none)
self._worker.stopped.connect(self._stopped)
self._worker.done.connect(self._done)
self._thread.started.connect(self._worker.do_work)
def filter_changed(self, text):
self.model_proxy.setFilterFixedString(text)
def stop_and_reset(self, reset_method=None):
if self.keep_running:
self.scheduled_call = reset_method or self.initialize
self.keep_running = False
else:
self.initialize()
def check_preconditions(self):
"""Check whether there is sufficient data for ranking."""
return True
def on_selection_changed(self, selected, deselected):
"""
Set the new visualization in the widget when the user select a
row in the table.
If derived class does not reimplement this, the table gives the
information but the user can't click it to select the visualization.
Args:
selected: the index of the selected item
deselected: the index of the previously selected item
"""
pass
def iterate_states(self, initial_state):
"""
Generate all possible states (e.g. attribute combinations) for the
given data. The content of the generated states is specific to the
visualization.
This method must be defined in the derived classes.
Args:
initial_state: initial state; None if this is the first call
"""
raise NotImplementedError
def state_count(self):
"""
Return the number of states for the progress bar.
Derived classes should implement this to ensure the proper behaviour of
the progress bar"""
return 0
def compute_score(self, state):
"""
Abstract method for computing the score for the given state. Smaller
scores are better.
Args:
state: the state, e.g. the combination of attributes as generated
by :obj:`state_count`.
"""
raise NotImplementedError
def bar_length(self, score):
"""Compute the bar length (between 0 and 1) corresponding to the score.
Return `None` if the score cannot be normalized.
"""
return None
def row_for_state(self, score, state):
"""
Abstract method that return the items that are inserted into the table.
Args:
score: score, computed by :obj:`compute_score`
state: the state, e.g. combination of attributes
"""
raise NotImplementedError
def _select_first_if_none(self):
if not self.rank_table.selectedIndexes():
self.rank_table.selectRow(0)
def _done(self):
self.button.setText("Finished")
self.button.setEnabled(False)
self.keep_running = False
self.saved_state = None
def _stopped(self):
self.update_timer.stop()
self.progressBarFinished()
self._update_model()
self.stopped()
if self.scheduled_call:
self.scheduled_call()
def _update(self):
self._update_model()
self._update_progress()
def _update_progress(self):
self.progressBarSet(int(self.saved_progress * 100 / max(1, self.state_count())))
def _update_model(self):
try:
while True:
pos, row_items = self.add_to_model.get_nowait()
self.rank_model.insertRow(pos, row_items)
except queue.Empty:
pass
def toggle(self):
"""Start or pause the computation."""
self.keep_running = not self.keep_running
if self.keep_running:
self.button.setText("Pause")
self.progressBarInit()
self.update_timer.start()
self.before_running()
self._thread.start()
else:
self.button.setText("Continue")
self._thread.quit()
# Need to sync state (the worker must read the keep_running
# state and stop) for reliable restart.
self._thread.wait()
def before_running(self):
"""Code that is run before running vizrank in its own thread"""
pass
def stopped(self):
"""Code that is run after stopping the vizrank thread"""
pass
class OWFilter(widget.OWWidget):
name = "Filter"
icon = 'icons/Filter.svg'
description = "Filter cells/genes"
priority = 210
class Inputs:
data = widget.Input("Data", Orange.data.Table)
class Outputs:
data = widget.Output("Data", Orange.data.Table)
class Warning(widget.OWWidget.Warning):
sampling_in_effect = widget.Msg("Too many data points to display.\n"
"Sampling {} of {} data points.")
class Error(widget.OWWidget.Error):
invalid_range = widget.Msg(
"Negative values in input data.\n"
"This filter is only defined for non-negative values.")
invalid_domain = widget.Msg("Invalid domain\n"
"Domain contains non numeric columns.")
#: Filter mode.
#: Filter out rows/columns or 'zap' data values in range.
Cells, Genes, Data = Cells, Genes, Data
settings_version = 3
#: The selected filter mode
selected_filter_type = settings.Setting(Cells) # type: int
#: Selected filter statistics / QC measure indexed by filter_type
selected_filter_metric = settings.Setting(TotalCounts) # type: int
#: Augment the violin plot with a dot plot (strip plot) of the (non-zero)
#: measurement counts in Cells/Genes mode or data matrix values in Data
#: mode.
display_dotplot = settings.Setting(True) # type: bool
#: Is min/max range selection enable
limit_lower_enabled = settings.Setting(True) # type: bool
limit_upper_enabled = settings.Setting(True) # type: bool
#: The lower and upper selection limit for each filter type
thresholds = settings.Setting({
(Cells, DetectionCount): (0, 2**31 - 1),
(Cells, TotalCounts): (0, 2**31 - 1),
(Genes, DetectionCount): (0, 2**31 - 1),
(Genes, TotalCounts): (0, 2**31 - 1),
(Data, -1): (0.0, 2.0**31 - 1)
}) # type: Dict[Tuple[int, int], Tuple[float, float]]
#: Plot scale: 'Linear' or 'Log1p'
scale = settings.Setting(Scale.Linear.name) # type: str
auto_commit = settings.Setting(True) # type: bool
def __init__(self):
super().__init__()
self.data = None # type: Optional[Orange.data.Table]
self._state = None # type: Optional[_FilterData]
box = gui.widgetBox(self.controlArea, "Info")
self._info = QLabel(box, wordWrap=True)
self._info.setText("No data in input\n")
box.layout().addWidget(self._info)
box = gui.widgetBox(self.controlArea, "Filter Type", spacing=-1)
rbg = QButtonGroup(box, exclusive=True)
layout = QHBoxLayout()
layout.setContentsMargins(0, 0, 0, 0)
for id_ in [Cells, Genes, Data]:
name, _, tip = FilterInfo[id_]
b = QRadioButton(name,
toolTip=tip,
checked=id_ == self.selected_filter_type)
rbg.addButton(b, id_)
layout.addWidget(b, stretch=10, alignment=Qt.AlignCenter)
box.layout().addLayout(layout)
rbg.buttonClicked[int].connect(self.set_filter_type)
self.filter_metric_cb = gui.comboBox(
box,
self,
"selected_filter_metric",
callback=self._update_metric,
enabled=self.selected_filter_type != Data)
for id_ in [DetectionCount, TotalCounts]:
text, ttip = MeasureInfo[id_]
self.filter_metric_cb.addItem(text)
idx = self.filter_metric_cb.count() - 1
self.filter_metric_cb.setItemData(idx, ttip, Qt.ToolTipRole)
self.filter_metric_cb.setCurrentIndex(self.selected_filter_metric)
form = QFormLayout(labelAlignment=Qt.AlignLeft,
formAlignment=Qt.AlignLeft,
fieldGrowthPolicy=QFormLayout.AllNonFixedFieldsGrow)
self._filter_box = box = gui.widgetBox(
self.controlArea, "Filter", orientation=form) # type: QGroupBox
self.threshold_stacks = (
QStackedWidget(enabled=self.limit_lower_enabled),
QStackedWidget(enabled=self.limit_upper_enabled),
)
finfo = np.finfo(np.float64)
for filter_ in [Cells, Genes, Data]:
if filter_ in {Cells, Genes}:
minimum = 0.0
ndecimals = 1
metric = self.selected_filter_metric
else:
minimum = finfo.min
ndecimals = 3
metric = -1
spinlower = QDoubleSpinBox(
self,
minimum=minimum,
maximum=finfo.max,
decimals=ndecimals,
keyboardTracking=False,
)
spinupper = QDoubleSpinBox(
self,
minimum=minimum,
maximum=finfo.max,
decimals=ndecimals,
keyboardTracking=False,
)
lower, upper = self.thresholds.get((filter_, metric), (0, 0))
spinlower.setValue(lower)
spinupper.setValue(upper)
self.threshold_stacks[0].addWidget(spinlower)
self.threshold_stacks[1].addWidget(spinupper)
spinlower.valueChanged.connect(self._limitchanged)
spinupper.valueChanged.connect(self._limitchanged)
self.threshold_stacks[0].setCurrentIndex(self.selected_filter_type)
self.threshold_stacks[1].setCurrentIndex(self.selected_filter_type)
self.limit_lower_enabled_cb = cb = QCheckBox(
"Min", checked=self.limit_lower_enabled)
cb.toggled.connect(self.set_lower_limit_enabled)
cb.setAttribute(Qt.WA_LayoutUsesWidgetRect, True)
form.addRow(cb, self.threshold_stacks[0])
self.limit_upper_enabled_cb = cb = QCheckBox(
"Max", checked=self.limit_upper_enabled)
cb.toggled.connect(self.set_upper_limit_enabled)
cb.setAttribute(Qt.WA_LayoutUsesWidgetRect, True)
form.addRow(cb, self.threshold_stacks[1])
box = gui.widgetBox(self.controlArea, "Plot Options")
self._showpoints = gui.checkBox(box,
self,
"display_dotplot",
"Show data points",
callback=self._update_dotplot)
self.log_scale_cb = QCheckBox("Log scale",
checked=self.scale == Scale.Log1p.name)
self.log_scale_cb.toggled[bool].connect(
lambda state: self.set_filter_scale(Scale.Log1p
if state else Scale.Linear))
box.layout().addWidget(self.log_scale_cb)
self.controlArea.layout().addStretch(10)
gui.auto_commit(self.controlArea, self, "auto_commit", "Commit")
self._view = pg.GraphicsView()
self._view.enableMouse(False)
self._view.setAntialiasing(True)
self._plot = plot = ViolinPlot()
self._plot.setDataPointsVisible(self.display_dotplot)
self._plot.setSelectionMode(
(ViolinPlot.Low if self.limit_lower_enabled else 0)
| (ViolinPlot.High if self.limit_upper_enabled else 0))
self._plot.setRange(QRectF(-1., 0., 2., 1.))
self._plot.selectionEdited.connect(self._limitchanged_plot)
self._view.setCentralWidget(self._plot)
bottom = self._plot.getAxis("bottom") # type: pg.AxisItem
bottom.hide()
plot.setMouseEnabled(False, False)
plot.hideButtons()
self.mainArea.layout().addWidget(self._view)
# Coalescing commit timer
self._committimer = QTimer(self, singleShot=True)
self._committimer.timeout.connect(self.commit)
self.addAction(
QAction("Select All",
self,
shortcut=QKeySequence.SelectAll,
triggered=self._select_all))
self._setup_axes()
def sizeHint(self):
sh = super().sizeHint() # type: QSize
return sh.expandedTo(QSize(800, 600))
def set_filter_type(self, type_):
if self.selected_filter_type != type_:
assert type_ in (Cells, Genes, Data), str(type_)
self.selected_filter_type = type_
self.threshold_stacks[0].setCurrentIndex(type_)
self.threshold_stacks[1].setCurrentIndex(type_)
self.filter_metric_cb.setEnabled(type_ != Data)
self._setup_axes()
if self.data is not None:
self._setup(self.data, type_)
self._schedule_commit()
def filter_type(self):
return self.selected_filter_type
def set_filter_scale(self, scale):
# type: (Scale) -> None
if self.scale != scale:
self.scale = scale.name
self.log_scale_cb.setChecked(scale == Scale.Log1p)
self._update_scale()
def filter_scale(self):
return Scale[self.scale]
def _update_metric(self):
self._update_scale()
if self.data is not None:
self._setup(
self.data,
self.selected_filter_type,
)
def set_upper_limit_enabled(self, enabled):
if enabled != self.limit_upper_enabled:
self.limit_upper_enabled = enabled
self.threshold_stacks[1].setEnabled(enabled)
self.limit_upper_enabled_cb.setChecked(enabled)
self._update_filter()
self._schedule_commit()
def set_lower_limit_enabled(self, enabled):
if enabled != self.limit_lower_enabled:
self.limit_lower_enabled = enabled
self.threshold_stacks[0].setEnabled(enabled)
self.limit_lower_enabled_cb.setChecked(enabled)
self._update_filter()
self._schedule_commit()
def _update_filter(self):
mode = 0
if self.limit_lower_enabled:
mode |= ViolinPlot.Low
if self.limit_upper_enabled:
mode |= ViolinPlot.High
self._plot.setSelectionMode(mode)
self._update_info()
self._schedule_commit()
def _is_filter_enabled(self):
return self.limit_lower_enabled or self.limit_upper_enabled
@Inputs.data
def set_data(self, data):
# type: (Optional[Orange.data.Table]) -> None
self.clear()
if data is not None and \
any(type(v) is not Orange.data.ContinuousVariable
for v in data.domain.attributes):
self.Error.invalid_domain()
data = None
if data is not None and np.any(data.X < 0):
self.Error.invalid_range()
data = None
self.data = data
if data is not None:
self._setup(data, self.filter_type())
self.unconditional_commit()
def clear(self):
self.data = None
self._state = None
self._plot.clear()
# reset the plot range
self._plot.setRange(QRectF(-1., 0., 2., 1.))
self._update_info()
self.Warning.clear()
self.Error.clear()
def _update_info(self):
text = []
if self.data is None:
text += ["No data on input.\n"]
else:
N, M = len(self.data), len(self.data.domain.attributes)
text = []
text += [
"Data with {N} cell{Np} and {M} gene{Mp}".format(
N=N,
Np="s" if N != 1 else "",
M=M,
Mp="s" if N != 1 else "")
]
if self._is_filter_enabled() and \
self.filter_type() in [Cells, Genes]:
counts = self._state.x
mask = np.ones(counts.shape, dtype=bool)
if self.limit_lower_enabled:
mask &= self.limit_lower <= counts
if self.limit_upper_enabled:
mask &= counts <= self.limit_upper
n = np.count_nonzero(mask)
subject = "cell" if self.filter_type() == Cells else "gene"
if n == 0:
text += ["All {}s filtered out".format(subject)]
else:
text += [
"{} {subject}{s} in selection".format(
n, subject=subject, s="s" if n != 1 else "")
]
else:
text += [""]
self._info.setText("\n".join(text))
def _select_all(self):
self.limit_lower = 0
self.limit_upper = 2**31 - 1
self._limitchanged()
def _setup_axes(self):
# Setup the plot axes and title
filter_type = self.filter_type()
info = FilterInfo[filter_type]
_, title, _, *_ = info
if filter_type in [Cells, Genes]:
measure = self.selected_filter_metric
else:
measure = None
if filter_type == Cells and measure == TotalCounts:
axis_label = "Total counts (library size)"
elif filter_type == Cells and measure == DetectionCount:
axis_label = "Number of expressed genes"
elif filter_type == Genes and measure == TotalCounts:
axis_label = "Total counts"
elif filter_type == Genes and measure == DetectionCount:
# TODO: Too long
axis_label = "Number of cells a gene is expressed in"
elif filter_type == Data:
axis_label = "Gene Expression"
ax = self._plot.getAxis("left")
if self.filter_scale() == Scale.Log1p:
axis_label = "1 + '{}' <i>(in log scale)</i>".format(axis_label)
ax.setLabel(axis_label)
ax.setLogMode(True)
else:
ax.setLogMode(False)
ax.setLabel(axis_label)
# Reset the tick text area width
ax.textWidth = 30
ax.setWidth(None)
self._plot.setTitle(title)
def _setup(self, data, filter_type):
self._plot.clear()
self._state = None
self._setup_axes()
span = -1.0 # data span
measure = self.selected_filter_metric if filter_type != Data else None
state = _FilterData()
if filter_type in [Cells, Genes]:
if filter_type == Cells:
axis = 1
else:
axis = 0
if measure == TotalCounts:
counts = np.nansum(data.X, axis=axis)
else:
mask = (data.X != 0) & (np.isfinite(data.X))
counts = np.count_nonzero(mask, axis=axis)
x = counts
self.Warning.sampling_in_effect.clear()
elif filter_type == Data:
x = data.X.ravel()
x = x[np.isfinite(x)]
x = x[x != 0]
MAX_DISPLAY_SIZE = 20000
if x.size > MAX_DISPLAY_SIZE:
self.Warning.sampling_in_effect(MAX_DISPLAY_SIZE, x.size)
# tails to preserve exactly
tails = 1
assert x.flags.owndata
x.partition(tails - 1)
xrest = x[tails:]
xrest.partition(xrest.size - tails)
x1, x2, x3 = x[:tails], x[tails:x.size - tails], x[x.size -
tails:]
assert x1.size + x2.size + x3.size == x.size
x2 = np.random.RandomState(0x667).choice(
x2,
size=MAX_DISPLAY_SIZE - 2 * tails,
replace=False,
)
x = np.r_[x1, x2, x3]
else:
self.Warning.sampling_in_effect.clear()
else:
assert False
state.x = x
scale = self.filter_scale()
if scale == Scale.Log1p:
scale_transform = log1p
scale_transform_inv = expm1
else:
scale_transform = lambda x: x
scale_transform_inv = scale_transform
state.transform = scale_transform
state.transform_inv = scale_transform_inv
if x.size:
xmin, xmax = np.min(x), np.max(x)
else:
xmin = xmax = 0., 1.
state.xmin, state.xmax = xmin, xmax
xs = scale_transform(x)
xs = xs[np.isfinite(xs)]
state.xt = xs
if xs.size:
xsmin, xsmax = np.min(xs), np.max(xs)
# find effective xmin, xmax (valid in both original and transformed
# space
xmin_, xmax_ = scale_transform_inv([xsmin, xsmax])
xmin, xmax = max(xmin, xmin_), min(xmax, xmax_)
lower = np.clip(self.limit_lower, xmin, xmax)
upper = np.clip(self.limit_upper, xmin, xmax)
else:
xmin, xmax = 0., 1.
lower, upper = 0., 1.
state.xtmin, state.xtmax = xsmin, xsmax
spinlow = self.threshold_stacks[0].widget(filter_type)
spinhigh = self.threshold_stacks[1].widget(filter_type)
if filter_type == Data or measure == TotalCounts:
span = xmax - xmin
if span > 0:
ndecimals = max(4 - int(np.floor(np.log10(span))), 1)
else:
ndecimals = 1
else:
ndecimals = 1
# Round effective bounds (spin <=> plot cut lines)
lower = round(lower, ndecimals)
upper = round(upper, ndecimals)
if xs.size > 0:
# TODO: Need correction for lower bounded distribution (counts)
# Use reflection around 0, but gaussian_kde does not provide
# sufficient flexibility w.r.t bandwidth selection.
self._plot.setData(xs, 1000)
self._plot.setBoundary(*scale_transform([lower, upper]))
spinlow.setDecimals(ndecimals)
self.limit_lower = lower
spinhigh.setDecimals(ndecimals)
self.limit_upper = upper
self._state = state
self._update_info()
def _update_dotplot(self):
self._plot.setDataPointsVisible(self.display_dotplot)
def current_filter_thresholds(self):
if self.selected_filter_type in {Cells, Genes}:
metric = self.selected_filter_metric
else:
metric = -1
return self.thresholds[self.selected_filter_type, metric]
def set_current_filter_thesholds(self, lower, upper):
if self.selected_filter_type in {Cells, Genes}:
metric = self.selected_filter_metric
else:
metric = -1
self.thresholds[self.selected_filter_type, metric] = (lower, upper)
def _update_scale(self):
self._setup_axes()
if self.data is not None:
self._setup(self.data, self.filter_type())
@property
def limit_lower(self):
return self.current_filter_thresholds()[0]
@limit_lower.setter
def limit_lower(self, value):
_, upper = self.current_filter_thresholds()
self.set_current_filter_thesholds(value, upper)
stacklower, _ = self.threshold_stacks
sb = stacklower.widget(self.selected_filter_type)
# prevent changes due to spin box rounding
sb.setValue(value)
@property
def limit_upper(self):
return self.current_filter_thresholds()[1]
@limit_upper.setter
def limit_upper(self, value):
lower, _ = self.current_filter_thresholds()
self.set_current_filter_thesholds(lower, value)
_, stackupper = self.threshold_stacks
sb = stackupper.widget(self.selected_filter_type)
sb.setValue(value)
@Slot()
def _limitchanged(self):
# Low/high limit changed via the spin boxes
stacklow, stackhigh = self.threshold_stacks
filter_ = self.selected_filter_type
lower = stacklow.widget(filter_).value()
upper = stackhigh.widget(filter_).value()
self.set_current_filter_thesholds(lower, upper)
state = self._state
if state is not None and state.x.size:
xmin, xmax = state.xmin, state.xmax
lower = np.clip(lower, xmin, xmax)
upper = np.clip(upper, xmin, xmax)
lower, upper = state.transform([lower, upper])
self._plot.setBoundary(lower, upper)
# TODO: Only when the actual selection/filter mask changes
self._schedule_commit()
self._update_info()
def _limitchanged_plot(self):
# Low/high limit changed via the plot
if self._state is not None:
state = self._state
newlower_, newupper_ = self._plot.boundary()
newlower, newupper = state.transform_inv([newlower_, newupper_])
filter_ = self.selected_filter_type
lower, upper = self.current_filter_thresholds()
stacklow, stackhigh = self.threshold_stacks
spin_lower = stacklow.widget(filter_)
spin_upper = stackhigh.widget(filter_)
# do rounding to match the spin box's precision
if self.limit_lower_enabled:
newlower = round(newlower, spin_lower.decimals())
else:
newlower = lower
if self.limit_upper_enabled:
newupper = round(newupper, spin_upper.decimals())
else:
newupper = upper
if self.limit_lower_enabled and newlower != lower:
self.limit_lower = newlower
if self.limit_upper_enabled and newupper != upper:
self.limit_upper = newupper
newlower_, newupper_ = state.transform([newlower, newupper])
self._plot.setBoundary(newlower_, newupper_)
# TODO: Only when the actual selection/filter mask changes
self._schedule_commit()
self._update_info()
def _schedule_commit(self):
self._committimer.start()
def commit(self):
self._committimer.stop()
data = self.data
if data is not None and self._is_filter_enabled():
if self.filter_type() in [Cells, Genes]:
state = self._state
assert state is not None
counts = state.x
cmax = self.limit_upper
cmin = self.limit_lower
mask = np.ones(counts.shape, dtype=bool)
if self.limit_lower_enabled:
mask &= cmin <= counts
if self.limit_upper_enabled:
mask &= counts <= cmax
if self.filter_type() == Cells:
assert counts.size == len(data)
data = data[mask]
else:
assert counts.size == len(data.domain.attributes)
atts = [
v for v, m in zip(data.domain.attributes, mask) if m
]
data = data.from_table(
Orange.data.Domain(atts, data.domain.class_vars,
data.domain.metas), data)
if len(data) == 0 or \
len(data.domain) + len(data.domain.metas) == 0:
data = None
elif self.filter_type() == Data:
dmin, dmax = self.limit_lower, self.limit_upper
data = data.copy()
assert data.X.base is None
mask = None
if self.limit_lower_enabled:
mask = data.X < dmin
if self.limit_upper_enabled:
if mask is not None:
mask |= data.X > dmax
else:
mask = data.X < dmax
data.X[mask] = 0.0
else:
assert False
self.Outputs.data.send(data)
def onDeleteWidget(self):
self.clear()
self._plot.close()
super().onDeleteWidget()
@classmethod
def migrate_settings(cls, settings, version):
if (version is None or version < 2) and \
("limit_lower" in settings and "limit_upper" in settings):
# v2 changed limit_lower, limit_upper to per filter limits stored
# in a single dict
lower = settings.pop("limit_lower")
upper = settings.pop("limit_upper")
settings["thresholds"] = {
(Cells, TotalCounts): (lower, upper),
(Cells, DetectionCount): (lower, upper),
(Genes, TotalCounts): (lower, upper),
(Genes, DetectionCount): (lower, upper),
(Data, -1): (lower, upper),
}
if version == 2:
thresholds = settings["thresholds"]
c = thresholds.pop(Cells)
g = thresholds.pop(Genes)
d = thresholds.pop(Data)
thresholds = {
(Cells, TotalCounts): c,
(Cells, DetectionCount): c,
(Genes, TotalCounts): g,
(Genes, DetectionCount): g,
(Data, -1): d,
}
settings["thresholds"] = thresholds
class OWMDS(OWDataProjectionWidget):
name = "MDS"
description = "Two-dimensional data projection by multidimensional " \
"scaling constructed from a distance matrix."
icon = "icons/MDS.svg"
keywords = ["multidimensional scaling", "multi dimensional scaling"]
class Inputs(OWDataProjectionWidget.Inputs):
distances = Input("Distances", DistMatrix)
settings_version = 3
#: Initialization type
PCA, Random, Jitter = 0, 1, 2
#: Refresh rate
RefreshRate = [
("Every iteration", 1),
("Every 5 steps", 5),
("Every 10 steps", 10),
("Every 25 steps", 25),
("Every 50 steps", 50),
("None", -1)
]
#: Runtime state
Running, Finished, Waiting = 1, 2, 3
max_iter = settings.Setting(300)
initialization = settings.Setting(PCA)
refresh_rate = settings.Setting(3)
GRAPH_CLASS = OWMDSGraph
graph = SettingProvider(OWMDSGraph)
embedding_variables_names = ("mds-x", "mds-y")
class Error(OWDataProjectionWidget.Error):
not_enough_rows = Msg("Input data needs at least 2 rows")
matrix_too_small = Msg("Input matrix must be at least 2x2")
no_attributes = Msg("Data has no attributes")
mismatching_dimensions = \
Msg("Data and distances dimensions do not match.")
out_of_memory = Msg("Out of memory")
optimization_error = Msg("Error during optimization\n{}")
def __init__(self):
super().__init__()
#: Input dissimilarity matrix
self.matrix = None # type: Optional[DistMatrix]
#: Data table from the `self.matrix.row_items` (if present)
self.matrix_data = None # type: Optional[Table]
#: Input data table
self.signal_data = None
self.__invalidated = True
self.embedding = None
self.effective_matrix = None
self.__update_loop = None
# timer for scheduling updates
self.__timer = QTimer(self, singleShot=True, interval=0)
self.__timer.timeout.connect(self.__next_step)
self.__state = OWMDS.Waiting
self.__in_next_step = False
self.graph.pause_drawing_pairs()
self.size_model = self.gui.points_models[2]
self.size_model.order = \
self.gui.points_models[2].order[:1] \
+ ("Stress", ) + \
self.gui.points_models[2].order[1:]
# self._initialize()
def _add_controls(self):
self._add_controls_optimization()
super()._add_controls()
self.gui.add_control(
self._effects_box, gui.hSlider, "Show similar pairs:",
master=self.graph, value="connected_pairs", minValue=0,
maxValue=20, createLabel=False, callback=self._on_connected_changed
)
def _add_controls_optimization(self):
box = gui.vBox(self.controlArea, box=True)
self.runbutton = gui.button(box, self, "Run optimization",
callback=self._toggle_run)
gui.comboBox(box, self, "refresh_rate", label="Refresh: ",
orientation=Qt.Horizontal,
items=[t for t, _ in OWMDS.RefreshRate],
callback=self.__invalidate_refresh)
hbox = gui.hBox(box, margin=0)
gui.button(hbox, self, "PCA", callback=self.do_PCA)
gui.button(hbox, self, "Randomize", callback=self.do_random)
gui.button(hbox, self, "Jitter", callback=self.do_jitter)
def set_data(self, data):
"""Set the input dataset.
Parameters
----------
data : Optional[Table]
"""
if data is not None and len(data) < 2:
self.Error.not_enough_rows()
data = None
else:
self.Error.not_enough_rows.clear()
self.signal_data = data
@Inputs.distances
def set_disimilarity(self, matrix):
"""Set the dissimilarity (distance) matrix.
Parameters
----------
matrix : Optional[Orange.misc.DistMatrix]
"""
if matrix is not None and len(matrix) < 2:
self.Error.matrix_too_small()
matrix = None
else:
self.Error.matrix_too_small.clear()
self.matrix = matrix
self.matrix_data = matrix.row_items if matrix is not None else None
def clear(self):
super().clear()
self.embedding = None
self.graph.set_effective_matrix(None)
self.__set_update_loop(None)
self.__state = OWMDS.Waiting
def _initialize(self):
matrix_existed = self.effective_matrix is not None
effective_matrix = self.effective_matrix
self.__invalidated = True
self.data = None
self.effective_matrix = None
self.closeContext()
self.clear_messages()
# if no data nor matrix is present reset plot
if self.signal_data is None and self.matrix is None:
self.clear()
self.init_attr_values()
return
if self.signal_data is not None and self.matrix is not None and \
len(self.signal_data) != len(self.matrix):
self.Error.mismatching_dimensions()
self.clear()
self.init_attr_values()
return
if self.signal_data is not None:
self.data = self.signal_data
elif self.matrix_data is not None:
self.data = self.matrix_data
if self.matrix is not None:
self.effective_matrix = self.matrix
if self.matrix.axis == 0 and self.data is self.matrix_data:
self.data = None
elif self.data.domain.attributes:
preprocessed_data = MDS().preprocess(self.data)
self.effective_matrix = Euclidean(preprocessed_data)
else:
self.Error.no_attributes()
self.clear()
self.init_attr_values()
return
self.init_attr_values()
self.openContext(self.data)
self.__invalidated = not (matrix_existed and
self.effective_matrix is not None and
np.array_equal(effective_matrix,
self.effective_matrix))
if self.__invalidated:
self.clear()
self.graph.set_effective_matrix(self.effective_matrix)
def _toggle_run(self):
if self.__state == OWMDS.Running:
self.stop()
self._invalidate_output()
else:
self.start()
def start(self):
if self.__state == OWMDS.Running:
return
elif self.__state == OWMDS.Finished:
# Resume/continue from a previous run
self.__start()
elif self.__state == OWMDS.Waiting and \
self.effective_matrix is not None:
self.__start()
def stop(self):
if self.__state == OWMDS.Running:
self.__set_update_loop(None)
def __start(self):
self.graph.pause_drawing_pairs()
X = self.effective_matrix
init = self.embedding
# number of iterations per single GUI update step
_, step_size = OWMDS.RefreshRate[self.refresh_rate]
if step_size == -1:
step_size = self.max_iter
def update_loop(X, max_iter, step, init):
"""
return an iterator over successive improved MDS point embeddings.
"""
# NOTE: this code MUST NOT call into QApplication.processEvents
done = False
iterations_done = 0
oldstress = np.finfo(np.float).max
init_type = "PCA" if self.initialization == OWMDS.PCA else "random"
while not done:
step_iter = min(max_iter - iterations_done, step)
mds = MDS(
dissimilarity="precomputed", n_components=2,
n_init=1, max_iter=step_iter,
init_type=init_type, init_data=init
)
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore", ".*double_scalars.*", RuntimeWarning)
mdsfit = mds(X)
iterations_done += step_iter
embedding, stress = mdsfit.embedding_, mdsfit.stress_
emb_norm = np.sqrt(np.sum(embedding ** 2, axis=1)).sum()
if emb_norm > 0:
stress /= emb_norm
if iterations_done >= max_iter \
or (oldstress - stress) < mds.params["eps"] \
or stress == 0:
done = True
init = embedding
oldstress = stress
yield embedding, mdsfit.stress_, iterations_done / max_iter
self.__set_update_loop(update_loop(X, self.max_iter, step_size, init))
self.progressBarInit(processEvents=None)
def __set_update_loop(self, loop):
"""
Set the update `loop` coroutine.
The `loop` is a generator yielding `(embedding, stress, progress)`
tuples where `embedding` is a `(N, 2) ndarray` of current updated
MDS points, `stress` is the current stress and `progress` a float
ratio (0 <= progress <= 1)
If an existing update coroutine loop is already in place it is
interrupted (i.e. closed).
.. note::
The `loop` must not explicitly yield control flow to the event
loop (i.e. call `QApplication.processEvents`)
"""
if self.__update_loop is not None:
self.__update_loop.close()
self.__update_loop = None
self.progressBarFinished(processEvents=None)
self.__update_loop = loop
if loop is not None:
self.setBlocking(True)
self.progressBarInit(processEvents=None)
self.setStatusMessage("Running")
self.runbutton.setText("Stop")
self.__state = OWMDS.Running
self.__timer.start()
else:
self.setBlocking(False)
self.setStatusMessage("")
self.runbutton.setText("Start")
self.__state = OWMDS.Finished
self.__timer.stop()
def __next_step(self):
if self.__update_loop is None:
return
assert not self.__in_next_step
self.__in_next_step = True
loop = self.__update_loop
self.Error.out_of_memory.clear()
try:
embedding, _, progress = next(self.__update_loop)
assert self.__update_loop is loop
except StopIteration:
self.__set_update_loop(None)
self.unconditional_commit()
self.graph.resume_drawing_pairs()
except MemoryError:
self.Error.out_of_memory()
self.__set_update_loop(None)
self.graph.resume_drawing_pairs()
except Exception as exc:
self.Error.optimization_error(str(exc))
self.__set_update_loop(None)
self.graph.resume_drawing_pairs()
else:
self.progressBarSet(100.0 * progress, processEvents=None)
self.embedding = embedding
self.graph.update_coordinates()
# schedule next update
self.__timer.start()
self.__in_next_step = False
def do_PCA(self):
self.__invalidate_embedding(self.PCA)
def do_random(self):
self.__invalidate_embedding(self.Random)
def do_jitter(self):
self.__invalidate_embedding(self.Jitter)
def __invalidate_embedding(self, initialization=PCA):
def jitter_coord(part):
span = np.max(part) - np.min(part)
part += np.random.uniform(-span / 20, span / 20, len(part))
# reset/invalidate the MDS embedding, to the default initialization
# (Random or PCA), restarting the optimization if necessary.
state = self.__state
if self.__update_loop is not None:
self.__set_update_loop(None)
if self.effective_matrix is None:
self.graph.reset_graph()
return
X = self.effective_matrix
if initialization == OWMDS.PCA:
self.embedding = torgerson(X)
elif initialization == OWMDS.Random:
self.embedding = np.random.rand(len(X), 2)
else:
jitter_coord(self.embedding[:, 0])
jitter_coord(self.embedding[:, 1])
self.setup_plot()
# restart the optimization if it was interrupted.
if state == OWMDS.Running:
self.__start()
def __invalidate_refresh(self):
state = self.__state
if self.__update_loop is not None:
self.__set_update_loop(None)
# restart the optimization if it was interrupted.
# TODO: decrease the max iteration count by the already
# completed iterations count.
if state == OWMDS.Running:
self.__start()
def handleNewSignals(self):
self._initialize()
if self.__invalidated:
self.graph.pause_drawing_pairs()
self.__invalidated = False
self.__invalidate_embedding()
self.cb_class_density.setEnabled(self.can_draw_density())
self.start()
else:
self.graph.update_point_props()
self.commit()
def _invalidate_output(self):
self.commit()
def _on_connected_changed(self):
self.graph.set_effective_matrix(self.effective_matrix)
self.graph.update_pairs(reconnect=True)
def setup_plot(self):
super().setup_plot()
if self.embedding is not None:
self.graph.update_pairs(reconnect=True)
def get_size_data(self):
if self.attr_size == "Stress":
return stress(self.embedding, self.effective_matrix)
else:
return super().get_size_data()
def get_embedding(self):
self.valid_data = np.ones(len(self.embedding), dtype=bool) \
if self.embedding is not None else None
return self.embedding
def _get_projection_data(self):
if self.embedding is None:
return None
if self.data is None:
x_name, y_name = self.embedding_variables_names
variables = ContinuousVariable(x_name), ContinuousVariable(y_name)
return Table(Domain(variables), self.embedding)
return super()._get_projection_data()
@classmethod
def migrate_settings(cls, settings_, version):
if version < 2:
settings_graph = {}
for old, new in (("label_only_selected", "label_only_selected"),
("symbol_opacity", "alpha_value"),
("symbol_size", "point_width"),
("jitter", "jitter_size")):
settings_graph[new] = settings_[old]
settings_["graph"] = settings_graph
settings_["auto_commit"] = settings_["autocommit"]
if version < 3:
if "connected_pairs" in settings_:
connected_pairs = settings_["connected_pairs"]
settings_["graph"]["connected_pairs"] = connected_pairs
@classmethod
def migrate_context(cls, context, version):
if version < 2:
domain = context.ordered_domain
n_domain = [t for t in context.ordered_domain if t[1] == 2]
c_domain = [t for t in context.ordered_domain if t[1] == 1]
context_values = {}
for _, old_val, new_val in ((domain, "color_value", "attr_color"),
(c_domain, "shape_value", "attr_shape"),
(n_domain, "size_value", "attr_size"),
(domain, "label_value", "attr_label")):
tmp = context.values[old_val]
if tmp[1] >= 0:
context_values[new_val] = (tmp[0], tmp[1] + 100)
elif tmp[0] != "Stress":
context_values[new_val] = None
else:
context_values[new_val] = tmp
context.values = context_values
if version < 3 and "graph" in context.values:
values = context.values
values["attr_color"] = values["graph"]["attr_color"]
values["attr_size"] = values["graph"]["attr_size"]
values["attr_shape"] = values["graph"]["attr_shape"]
values["attr_label"] = values["graph"]["attr_label"]
class QuickHelp(QTextBrowser):
#: Emitted when the shown text changes.
textChanged = Signal()
def __init__(self, *args, **kwargs):
QTextBrowser.__init__(self, *args, **kwargs)
self.setOpenExternalLinks(False)
self.setOpenLinks(False)
self.__text = ""
self.__permanentText = ""
self.__timer = QTimer(self, timeout=self.__on_timeout, singleShot=True)
self.anchorClicked.connect(self.__on_anchorClicked)
def showHelp(self, text, timeout=0):
"""
Show help for `timeout` milliseconds. if timeout is 0 then
show the text until it is cleared with clearHelp or showHelp is
called with an empty string.
"""
if self.__text != text:
self.__text = str(text)
self.__update()
self.textChanged.emit()
if timeout > 0:
self.__timer.start(timeout)
def clearHelp(self):
"""
Clear help text previously set with `showHelp`.
"""
self.__timer.stop()
self.showHelp("")
def showPermanentHelp(self, text):
"""
Set permanent help text. The text may be temporarily overridden
by showHelp but will be shown again when that is cleared.
"""
if self.__permanentText != text:
self.__permanentText = text
self.__update()
self.textChanged.emit()
def currentText(self):
"""
Return the current shown text.
"""
return self.__text or self.__permanentText
def __update(self):
if self.__text:
self.setHtml(self.__text)
else:
self.setHtml(self.__permanentText)
def __on_timeout(self):
if self.__text:
self.__text = ""
self.__update()
self.textChanged.emit()
def __on_anchorClicked(self, anchor):
ev = QuickHelpDetailRequestEvent(anchor.toString(), anchor)
QCoreApplication.postEvent(self, ev)
class TabBarWidget(QWidget):
"""
A tab bar widget using tool buttons as tabs.
"""
# TODO: A uniform size box layout.
currentChanged = Signal(int)
def __init__(self, parent=None, **kwargs):
QWidget.__init__(self, parent, **kwargs)
layout = QVBoxLayout()
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(0)
self.setLayout(layout)
self.setSizePolicy(QSizePolicy.Fixed,
QSizePolicy.Expanding)
self.__tabs = []
self.__currentIndex = -1
self.__changeOnHover = False
self.__iconSize = QSize(26, 26)
self.__group = QButtonGroup(self, exclusive=True)
self.__group.buttonPressed[QAbstractButton].connect(
self.__onButtonPressed
)
self.setMouseTracking(True)
self.__sloppyButton = None
self.__sloppyRegion = QRegion()
self.__sloppyTimer = QTimer(self, singleShot=True)
self.__sloppyTimer.timeout.connect(self.__onSloppyTimeout)
def setChangeOnHover(self, changeOnHover):
"""
If set to ``True`` the tab widget will change the current index when
the mouse hovers over a tab button.
"""
if self.__changeOnHover != changeOnHover:
self.__changeOnHover = changeOnHover
def changeOnHover(self):
"""
Does the current tab index follow the mouse cursor.
"""
return self.__changeOnHover
def count(self):
"""
Return the number of tabs in the widget.
"""
return len(self.__tabs)
def addTab(self, text, icon=None, toolTip=None):
"""
Add a new tab and return it's index.
"""
return self.insertTab(self.count(), text, icon, toolTip)
def insertTab(self, index, text, icon=None, toolTip=None):
"""
Insert a tab at `index`
"""
button = TabButton(self, objectName="tab-button")
button.setSizePolicy(QSizePolicy.Expanding,
QSizePolicy.Expanding)
button.setIconSize(self.__iconSize)
button.setMouseTracking(True)
self.__group.addButton(button)
button.installEventFilter(self)
tab = _Tab(text, icon, toolTip, button, None, None)
self.layout().insertWidget(index, button)
self.__tabs.insert(index, tab)
self.__updateTab(index)
if self.currentIndex() == -1:
self.setCurrentIndex(0)
return index
def removeTab(self, index):
"""
Remove a tab at `index`.
"""
if index >= 0 and index < self.count():
self.layout().takeItem(index)
tab = self.__tabs.pop(index)
self.__group.removeButton(tab.button)
tab.button.removeEventFilter(self)
if tab.button is self.__sloppyButton:
self.__sloppyButton = None
self.__sloppyRegion = QRegion()
tab.button.deleteLater()
if self.currentIndex() == index:
if self.count():
self.setCurrentIndex(max(index - 1, 0))
else:
self.setCurrentIndex(-1)
def setTabIcon(self, index, icon):
"""
Set the `icon` for tab at `index`.
"""
self.__tabs[index] = self.__tabs[index]._replace(icon=icon)
self.__updateTab(index)
def setTabToolTip(self, index, toolTip):
"""
Set `toolTip` for tab at `index`.
"""
self.__tabs[index] = self.__tabs[index]._replace(toolTip=toolTip)
self.__updateTab(index)
def setTabText(self, index, text):
"""
Set tab `text` for tab at `index`
"""
self.__tabs[index] = self.__tabs[index]._replace(text=text)
self.__updateTab(index)
def setTabPalette(self, index, palette):
"""
Set the tab button palette.
"""
self.__tabs[index] = self.__tabs[index]._replace(palette=palette)
self.__updateTab(index)
def setCurrentIndex(self, index):
"""
Set the current tab index.
"""
if self.__currentIndex != index:
self.__currentIndex = index
self.__sloppyRegion = QRegion()
self.__sloppyButton = None
if index != -1:
self.__tabs[index].button.setChecked(True)
self.currentChanged.emit(index)
def currentIndex(self):
"""
Return the current index.
"""
return self.__currentIndex
def button(self, index):
"""
Return the `TabButton` instance for index.
"""
return self.__tabs[index].button
def setIconSize(self, size):
if self.__iconSize != size:
self.__iconSize = size
for tab in self.__tabs:
tab.button.setIconSize(self.__iconSize)
def __updateTab(self, index):
"""
Update the tab button.
"""
tab = self.__tabs[index]
b = tab.button
if tab.text:
b.setText(tab.text)
if tab.icon is not None and not tab.icon.isNull():
b.setIcon(tab.icon)
if tab.palette:
b.setPalette(tab.palette)
def __onButtonPressed(self, button):
for i, tab in enumerate(self.__tabs):
if tab.button is button:
self.setCurrentIndex(i)
break
def __calcSloppyRegion(self, current):
"""
Given a current mouse cursor position return a region of the widget
where hover/move events should change the current tab only on a
timeout.
"""
p1 = current + QPoint(0, 2)
p2 = current + QPoint(0, -2)
p3 = self.pos() + QPoint(self.width()+10, 0)
p4 = self.pos() + QPoint(self.width()+10, self.height())
return QRegion(QPolygon([p1, p2, p3, p4]))
def __setSloppyButton(self, button):
"""
Set the current sloppy button (a tab button inside sloppy region)
and reset the sloppy timeout.
"""
if not button.isChecked():
self.__sloppyButton = button
delay = self.style().styleHint(QStyle.SH_Menu_SubMenuPopupDelay, None)
# The delay timeout is the same as used by Qt in the QMenu.
self.__sloppyTimer.start(delay)
else:
self.__sloppyTimer.stop()
def __onSloppyTimeout(self):
if self.__sloppyButton is not None:
button = self.__sloppyButton
self.__sloppyButton = None
if not button.isChecked():
index = [tab.button for tab in self.__tabs].index(button)
self.setCurrentIndex(index)
def eventFilter(self, receiver, event):
if event.type() == QEvent.MouseMove and \
isinstance(receiver, TabButton):
pos = receiver.mapTo(self, event.pos())
if self.__sloppyRegion.contains(pos):
self.__setSloppyButton(receiver)
else:
if not receiver.isChecked():
index = [tab.button for tab in self.__tabs].index(receiver)
self.setCurrentIndex(index)
#also update sloppy region if mouse is moved on the same icon
self.__sloppyRegion = self.__calcSloppyRegion(pos)
return QWidget.eventFilter(self, receiver, event)
def leaveEvent(self, event):
self.__sloppyButton = None
self.__sloppyRegion = QRegion()
return QWidget.leaveEvent(self, event)
class OWScatterPlot(OWDataProjectionWidget):
"""Scatterplot visualization with explorative analysis and intelligent
data visualization enhancements."""
name = 'Scatter Plot'
description = "Interactive scatter plot visualization with " \
"intelligent data visualization enhancements."
icon = "icons/ScatterPlot.svg"
priority = 140
keywords = []
class Inputs(OWDataProjectionWidget.Inputs):
features = Input("Features", AttributeList)
class Outputs(OWDataProjectionWidget.Outputs):
features = Output("Features", AttributeList, dynamic=False)
settings_version = 5
auto_sample = Setting(True)
attr_x = ContextSetting(None)
attr_y = ContextSetting(None)
tooltip_shows_all = Setting(True)
GRAPH_CLASS = OWScatterPlotGraph
graph = SettingProvider(OWScatterPlotGraph)
embedding_variables_names = None
xy_changed_manually = Signal(Variable, Variable)
class Warning(OWDataProjectionWidget.Warning):
missing_coords = Msg("Plot cannot be displayed because '{}' or '{}' "
"is missing for all data points.")
class Information(OWDataProjectionWidget.Information):
sampled_sql = Msg("Large SQL table; showing a sample.")
missing_coords = Msg(
"Points with missing '{}' or '{}' are not displayed")
def __init__(self):
self.attr_box: QGroupBox = None
self.xy_model: DomainModel = None
self.cb_attr_x: ComboBoxSearch = None
self.cb_attr_y: ComboBoxSearch = None
self.vizrank: ScatterPlotVizRank = None
self.vizrank_button: QPushButton = None
self.sampling: QGroupBox = None
self.sql_data = None # Orange.data.sql.table.SqlTable
self.attribute_selection_list = None # list of Orange.data.Variable
self.__timer = QTimer(self, interval=1200)
self.__timer.timeout.connect(self.add_data)
super().__init__()
# manually register Matplotlib file writers
self.graph_writers = self.graph_writers.copy()
for w in [MatplotlibFormat, MatplotlibPDFFormat]:
self.graph_writers.append(w)
def _add_controls(self):
self._add_controls_axis()
self._add_controls_sampling()
super()._add_controls()
self.gui.add_widget(self.gui.JitterNumericValues, self._effects_box)
self.gui.add_widgets([
self.gui.ShowGridLines, self.gui.ToolTipShowsAll,
self.gui.RegressionLine
], self._plot_box)
gui.checkBox(
self._plot_box,
self,
value="graph.orthonormal_regression",
label="Treat variables as independent",
callback=self.graph.update_regression_line,
tooltip=
"If checked, fit line to group (minimize distance from points);\n"
"otherwise fit y as a function of x (minimize vertical distances)",
disabledBy=self.cb_reg_line)
def _add_controls_axis(self):
common_options = dict(labelWidth=50,
orientation=Qt.Horizontal,
sendSelectedValue=True,
contentsLength=12,
searchable=True)
self.attr_box = gui.vBox(self.controlArea,
'Axes',
spacing=2 if gui.is_macstyle() else 8)
dmod = DomainModel
self.xy_model = DomainModel(dmod.MIXED, valid_types=dmod.PRIMITIVE)
self.cb_attr_x = gui.comboBox(
self.attr_box,
self,
"attr_x",
label="Axis x:",
callback=self.set_attr_from_combo,
model=self.xy_model,
**common_options,
)
self.cb_attr_y = gui.comboBox(
self.attr_box,
self,
"attr_y",
label="Axis y:",
callback=self.set_attr_from_combo,
model=self.xy_model,
**common_options,
)
vizrank_box = gui.hBox(self.attr_box)
self.vizrank, self.vizrank_button = ScatterPlotVizRank.add_vizrank(
vizrank_box, self, "Find Informative Projections", self.set_attr)
def _add_controls_sampling(self):
self.sampling = gui.auto_commit(self.controlArea,
self,
"auto_sample",
"Sample",
box="Sampling",
callback=self.switch_sampling,
commit=lambda: self.add_data(1))
self.sampling.setVisible(False)
@property
def effective_variables(self):
return [self.attr_x, self.attr_y
] if self.attr_x and self.attr_y else []
@property
def effective_data(self):
eff_var = self.effective_variables
if eff_var and self.attr_x.name == self.attr_y.name:
eff_var = [self.attr_x]
return self.data.transform(Domain(eff_var))
def _vizrank_color_change(self):
self.vizrank.initialize()
err_msg = ""
if self.data is None:
err_msg = "No data on input"
elif self.data.is_sparse():
err_msg = "Data is sparse"
elif len(self.xy_model) < 3:
err_msg = "Not enough features for ranking"
elif self.attr_color is None:
err_msg = "Color variable is not selected"
elif np.isnan(
self.data.get_column_view(
self.attr_color)[0].astype(float)).all():
err_msg = "Color variable has no values"
self.vizrank_button.setEnabled(not err_msg)
self.vizrank_button.setToolTip(err_msg)
def set_data(self, data):
super().set_data(data)
self._vizrank_color_change()
def findvar(name, iterable):
"""Find a Orange.data.Variable in `iterable` by name"""
for el in iterable:
if isinstance(el, Variable) and el.name == name:
return el
return None
# handle restored settings from < 3.3.9 when attr_* were stored
# by name
if isinstance(self.attr_x, str):
self.attr_x = findvar(self.attr_x, self.xy_model)
if isinstance(self.attr_y, str):
self.attr_y = findvar(self.attr_y, self.xy_model)
if isinstance(self.attr_label, str):
self.attr_label = findvar(self.attr_label, self.gui.label_model)
if isinstance(self.attr_color, str):
self.attr_color = findvar(self.attr_color, self.gui.color_model)
if isinstance(self.attr_shape, str):
self.attr_shape = findvar(self.attr_shape, self.gui.shape_model)
if isinstance(self.attr_size, str):
self.attr_size = findvar(self.attr_size, self.gui.size_model)
def check_data(self):
super().check_data()
self.__timer.stop()
self.sampling.setVisible(False)
self.sql_data = None
if isinstance(self.data, SqlTable):
if self.data.approx_len() < 4000:
self.data = Table(self.data)
else:
self.Information.sampled_sql()
self.sql_data = self.data
data_sample = self.data.sample_time(0.8, no_cache=True)
data_sample.download_data(2000, partial=True)
self.data = Table(data_sample)
self.sampling.setVisible(True)
if self.auto_sample:
self.__timer.start()
if self.data is not None and (len(self.data) == 0
or len(self.data.domain.variables) == 0):
self.data = None
def get_embedding(self):
self.valid_data = None
if self.data is None:
return None
x_data = self.get_column(self.attr_x, filter_valid=False)
y_data = self.get_column(self.attr_y, filter_valid=False)
if x_data is None or y_data is None:
return None
self.Warning.missing_coords.clear()
self.Information.missing_coords.clear()
self.valid_data = np.isfinite(x_data) & np.isfinite(y_data)
if self.valid_data is not None and not np.all(self.valid_data):
msg = self.Information if np.any(self.valid_data) else self.Warning
msg.missing_coords(self.attr_x.name, self.attr_y.name)
return np.vstack((x_data, y_data)).T
# Tooltip
def _point_tooltip(self, point_id, skip_attrs=()):
point_data = self.data[point_id]
xy_attrs = (self.attr_x, self.attr_y)
text = "<br/>".join(
escape('{} = {}'.format(var.name, point_data[var]))
for var in xy_attrs)
if self.tooltip_shows_all:
others = super()._point_tooltip(point_id, skip_attrs=xy_attrs)
if others:
text = "<b>{}</b><br/><br/>{}".format(text, others)
return text
def can_draw_regresssion_line(self):
return self.data is not None and \
self.data.domain is not None and \
self.attr_x.is_continuous and \
self.attr_y.is_continuous
def add_data(self, time=0.4):
if self.data and len(self.data) > 2000:
self.__timer.stop()
return
data_sample = self.sql_data.sample_time(time, no_cache=True)
if data_sample:
data_sample.download_data(2000, partial=True)
data = Table(data_sample)
self.data = Table.concatenate((self.data, data), axis=0)
self.handleNewSignals()
def init_attr_values(self):
super().init_attr_values()
data = self.data
domain = data.domain if data and len(data) else None
self.xy_model.set_domain(domain)
self.attr_x = self.xy_model[0] if self.xy_model else None
self.attr_y = self.xy_model[1] if len(self.xy_model) >= 2 \
else self.attr_x
def switch_sampling(self):
self.__timer.stop()
if self.auto_sample and self.sql_data:
self.add_data()
self.__timer.start()
def set_subset_data(self, subset_data):
self.warning()
if isinstance(subset_data, SqlTable):
if subset_data.approx_len() < AUTO_DL_LIMIT:
subset_data = Table(subset_data)
else:
self.warning("Data subset does not support large Sql tables")
subset_data = None
super().set_subset_data(subset_data)
# called when all signals are received, so the graph is updated only once
def handleNewSignals(self):
self.attr_box.setEnabled(True)
self.vizrank.setEnabled(True)
if self.attribute_selection_list and self.data is not None and \
self.data.domain is not None and \
all(attr in self.data.domain for attr
in self.attribute_selection_list):
self.attr_x, self.attr_y = self.attribute_selection_list[:2]
self.attr_box.setEnabled(False)
self.vizrank.setEnabled(False)
super().handleNewSignals()
if self._domain_invalidated:
self.graph.update_axes()
self._domain_invalidated = False
self.cb_reg_line.setEnabled(self.can_draw_regresssion_line())
@Inputs.features
def set_shown_attributes(self, attributes):
if attributes and len(attributes) >= 2:
self.attribute_selection_list = attributes[:2]
self._invalidated = self._invalidated \
or self.attr_x != attributes[0] \
or self.attr_y != attributes[1]
else:
self.attribute_selection_list = None
def set_attr(self, attr_x, attr_y):
if attr_x != self.attr_x or attr_y != self.attr_y:
self.attr_x, self.attr_y = attr_x, attr_y
self.attr_changed()
def set_attr_from_combo(self):
self.attr_changed()
self.xy_changed_manually.emit(self.attr_x, self.attr_y)
def attr_changed(self):
self.cb_reg_line.setEnabled(self.can_draw_regresssion_line())
self.setup_plot()
self.commit.deferred()
def get_axes(self):
return {"bottom": self.attr_x, "left": self.attr_y}
def colors_changed(self):
super().colors_changed()
self._vizrank_color_change()
@gui.deferred
def commit(self):
super().commit()
self.send_features()
def send_features(self):
features = [attr for attr in [self.attr_x, self.attr_y] if attr]
self.Outputs.features.send(AttributeList(features) or None)
def get_widget_name_extension(self):
if self.data is not None:
return "{} vs {}".format(self.attr_x.name, self.attr_y.name)
return None
def _get_send_report_caption(self):
return report.render_items_vert(
(("Color", self._get_caption_var_name(self.attr_color)),
("Label", self._get_caption_var_name(self.attr_label)),
("Shape", self._get_caption_var_name(self.attr_shape)),
("Size", self._get_caption_var_name(self.attr_size)),
("Jittering", (self.attr_x.is_discrete or self.attr_y.is_discrete
or self.graph.jitter_continuous)
and self.graph.jitter_size)))
@classmethod
def migrate_settings(cls, settings, version):
if version < 2 and "selection" in settings and settings["selection"]:
settings["selection_group"] = [(a, 1)
for a in settings["selection"]]
if version < 3:
if "auto_send_selection" in settings:
settings["auto_commit"] = settings["auto_send_selection"]
if "selection_group" in settings:
settings["selection"] = settings["selection_group"]
if version < 5:
if "graph" in settings and \
"jitter_continuous" not in settings["graph"]:
settings["graph"]["jitter_continuous"] = True
@classmethod
def migrate_context(cls, context, version):
values = context.values
if version < 3:
values["attr_color"] = values["graph"]["attr_color"]
values["attr_size"] = values["graph"]["attr_size"]
values["attr_shape"] = values["graph"]["attr_shape"]
values["attr_label"] = values["graph"]["attr_label"]
if version < 4:
if values["attr_x"][1] % 100 == 1 or values["attr_y"][1] % 100 == 1:
raise IncompatibleContext()
class OWtSNE(OWDataProjectionWidget):
name = "t-SNE"
description = "Two-dimensional data projection with t-SNE."
icon = "icons/TSNE.svg"
priority = 920
keywords = ["tsne"]
settings_version = 3
max_iter = Setting(300)
perplexity = Setting(30)
multiscale = Setting(False)
exaggeration = Setting(1)
pca_components = Setting(20)
normalize = Setting(True)
GRAPH_CLASS = OWtSNEGraph
graph = SettingProvider(OWtSNEGraph)
embedding_variables_names = ("t-SNE-x", "t-SNE-y")
#: Runtime state
Running, Finished, Waiting, Paused = 1, 2, 3, 4
class Error(OWDataProjectionWidget.Error):
not_enough_rows = Msg("Input data needs at least 2 rows")
constant_data = Msg("Input data is constant")
no_attributes = Msg("Data has no attributes")
out_of_memory = Msg("Out of memory")
optimization_error = Msg("Error during optimization\n{}")
no_valid_data = Msg("No projection due to no valid data")
def __init__(self):
super().__init__()
self.pca_data = None
self.projection = None
self.tsne_runner = None
self.tsne_iterator = None
self.__update_loop = None
# timer for scheduling updates
self.__timer = QTimer(self, singleShot=True, interval=1,
timeout=self.__next_step)
self.__state = OWtSNE.Waiting
self.__in_next_step = False
self.__draw_similar_pairs = False
def reset_needs_to_draw():
self.needs_to_draw = True
self.needs_to_draw = True
self.__timer_draw = QTimer(self, interval=2000,
timeout=reset_needs_to_draw)
def _add_controls(self):
self._add_controls_start_box()
super()._add_controls()
def _add_controls_start_box(self):
box = gui.vBox(self.controlArea, True)
form = QFormLayout(
labelAlignment=Qt.AlignLeft,
formAlignment=Qt.AlignLeft,
fieldGrowthPolicy=QFormLayout.AllNonFixedFieldsGrow,
verticalSpacing=10,
)
self.perplexity_spin = gui.spin(
box, self, "perplexity", 1, 500, step=1, alignment=Qt.AlignRight,
callback=self._params_changed
)
form.addRow("Perplexity:", self.perplexity_spin)
self.perplexity_spin.setEnabled(not self.multiscale)
form.addRow(gui.checkBox(
box, self, "multiscale", label="Preserve global structure",
callback=self._multiscale_changed
))
sbe = gui.hBox(self.controlArea, False, addToLayout=False)
gui.hSlider(
sbe, self, "exaggeration", minValue=1, maxValue=4, step=1,
callback=self._params_changed
)
form.addRow("Exaggeration:", sbe)
sbp = gui.hBox(self.controlArea, False, addToLayout=False)
gui.hSlider(
sbp, self, "pca_components", minValue=2, maxValue=50, step=1,
callback=self._invalidate_pca_projection
)
form.addRow("PCA components:", sbp)
self.normalize_cbx = gui.checkBox(
box, self, "normalize", "Normalize data",
callback=self._invalidate_pca_projection,
)
form.addRow(self.normalize_cbx)
box.layout().addLayout(form)
gui.separator(box, 10)
self.runbutton = gui.button(box, self, "Run", callback=self._toggle_run)
def _invalidate_pca_projection(self):
self.pca_data = None
self._params_changed()
def _params_changed(self):
self.__state = OWtSNE.Finished
self.__set_update_loop(None)
def _multiscale_changed(self):
self.perplexity_spin.setEnabled(not self.multiscale)
self._params_changed()
def check_data(self):
def error(err):
err()
self.data = None
super().check_data()
if self.data is not None:
if len(self.data) < 2:
error(self.Error.not_enough_rows)
elif not self.data.domain.attributes:
error(self.Error.no_attributes)
elif not self.data.is_sparse():
if np.all(~np.isfinite(self.data.X)):
error(self.Error.no_valid_data)
else:
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore", "Degrees of freedom .*", RuntimeWarning)
if np.nan_to_num(np.nanstd(self.data.X, axis=0)).sum() \
== 0:
error(self.Error.constant_data)
def get_embedding(self):
if self.data is None:
self.valid_data = None
return None
elif self.projection is None:
embedding = np.random.normal(size=(len(self.data), 2))
else:
embedding = self.projection.embedding.X
self.valid_data = np.ones(len(embedding), dtype=bool)
return embedding
def _toggle_run(self):
if self.__state == OWtSNE.Running:
self.stop()
self.commit()
elif self.__state == OWtSNE.Paused:
self.resume()
else:
self.start()
def start(self):
if not self.data or self.__state == OWtSNE.Running:
self.graph.update_coordinates()
elif self.__state in (OWtSNE.Finished, OWtSNE.Waiting):
self.__start()
def stop(self):
self.__state = OWtSNE.Paused
self.__set_update_loop(None)
def resume(self):
self.__set_update_loop(self.tsne_iterator)
def set_data(self, data: Table):
super().set_data(data)
if data is not None:
# PCA doesn't support normalization on sparse data, as this would
# require centering and normalizing the matrix
self.normalize_cbx.setDisabled(data.is_sparse())
if data.is_sparse():
self.normalize = False
self.normalize_cbx.setToolTip(
"Data normalization is not supported on sparse matrices."
)
else:
self.normalize_cbx.setToolTip("")
def pca_preprocessing(self):
"""Perform PCA preprocessing before passing off the data to t-SNE."""
if self.pca_data is not None:
return
projector = PCA(n_components=self.pca_components, random_state=0)
# If the normalization box is ticked, we'll add the `Normalize`
# preprocessor to PCA
if self.normalize:
projector.preprocessors += (preprocess.Normalize(),)
model = projector(self.data)
self.pca_data = model(self.data)
def __start(self):
self.pca_preprocessing()
self.needs_to_draw = True
# We call PCA through fastTSNE because it involves scaling. Instead of
# worrying about this ourselves, we'll let the library worry for us.
initialization = TSNE.default_initialization(
self.pca_data.X, n_components=2, random_state=0)
# Compute perplexity settings for multiscale
n_samples = self.pca_data.X.shape[0]
if self.multiscale:
perplexity = min((n_samples - 1) / 3, 50), min((n_samples - 1) / 3, 500)
else:
perplexity = self.perplexity
# Determine whether to use settings for large data sets
if n_samples > 10_000:
neighbor_method, gradient_method = "approx", "fft"
else:
neighbor_method, gradient_method = "exact", "bh"
# Set number of iterations to 0 - these will be run subsequently
self.projection = TSNE(
n_components=2, perplexity=perplexity, multiscale=self.multiscale,
early_exaggeration_iter=0, n_iter=0, initialization=initialization,
exaggeration=self.exaggeration, neighbors=neighbor_method,
negative_gradient_method=gradient_method, random_state=0,
theta=0.8,
)(self.pca_data)
self.tsne_runner = TSNERunner(
self.projection, step_size=20, exaggeration=self.exaggeration
)
self.tsne_iterator = self.tsne_runner.run_optimization()
self.__set_update_loop(self.tsne_iterator)
self.progressBarInit(processEvents=None)
def __set_update_loop(self, loop):
if self.__update_loop is not None:
if self.__state in (OWtSNE.Finished, OWtSNE.Waiting):
self.__update_loop.close()
self.__update_loop = None
self.progressBarFinished(processEvents=None)
self.__update_loop = loop
if loop is not None:
self.setBlocking(True)
self.progressBarInit(processEvents=None)
self.setStatusMessage("Running")
self.runbutton.setText("Stop")
self.__state = OWtSNE.Running
self.__timer.start()
self.__timer_draw.start()
else:
self.setBlocking(False)
self.setStatusMessage("")
if self.__state in (OWtSNE.Finished, OWtSNE.Waiting):
self.runbutton.setText("Start")
if self.__state == OWtSNE.Paused:
self.runbutton.setText("Resume")
self.__timer.stop()
self.__timer_draw.stop()
def __next_step(self):
if self.__update_loop is None:
return
assert not self.__in_next_step
self.__in_next_step = True
loop = self.__update_loop
self.Error.out_of_memory.clear()
self.Error.optimization_error.clear()
try:
projection, progress = next(self.__update_loop)
assert self.__update_loop is loop
except StopIteration:
self.__state = OWtSNE.Finished
self.__set_update_loop(None)
self.unconditional_commit()
except MemoryError:
self.Error.out_of_memory()
self.__state = OWtSNE.Finished
self.__set_update_loop(None)
except Exception as exc:
self.Error.optimization_error(str(exc))
self.__state = OWtSNE.Finished
self.__set_update_loop(None)
else:
self.progressBarSet(100.0 * progress, processEvents=None)
self.projection = projection
if progress == 1 or self.needs_to_draw:
self.graph.update_coordinates()
self.graph.update_density()
self.needs_to_draw = False
# schedule next update
self.__timer.start()
self.__in_next_step = False
def setup_plot(self):
super().setup_plot()
self.start()
def _get_projection_data(self):
if self.data is None:
return None
data = self.data.transform(
Domain(self.data.domain.attributes,
self.data.domain.class_vars,
self.data.domain.metas + self._get_projection_variables()))
data.metas[:, -2:] = self.get_embedding()
if self.projection is not None:
data.domain = Domain(
self.data.domain.attributes,
self.data.domain.class_vars,
self.data.domain.metas + self.projection.domain.attributes)
return data
def clear(self):
super().clear()
self.__state = OWtSNE.Waiting
self.__set_update_loop(None)
self.pca_data = None
self.projection = None
@classmethod
def migrate_settings(cls, settings, version):
if version < 3:
if "selection_indices" in settings:
settings["selection"] = settings["selection_indices"]
@classmethod
def migrate_context(cls, context, version):
if version < 3:
values = context.values
values["attr_color"] = values["graph"]["attr_color"]
values["attr_size"] = values["graph"]["attr_size"]
values["attr_shape"] = values["graph"]["attr_shape"]
values["attr_label"] = values["graph"]["attr_label"]
class WidgetManager(QObject):
"""
WidgetManager class is responsible for creation, tracking and deletion
of UI elements constituting an interactive workflow.
It does so by reacting to changes in the underlying workflow model,
creating and destroying the components when needed.
This is an abstract class, subclassed MUST reimplement at least
:func:`create_widget_for_node` and :func:`delete_widget_for_node`.
The widgets created with :func:`create_widget_for_node` will automatically
receive dispatched events:
* :data:`WorkflowEvent.InputLinkAdded` - when a new input link is
added to the workflow.
* :data:`LinkEvent.InputLinkRemoved` - when a input link is removed.
* :data:`LinkEvent.OutputLinkAdded` - when a new output link is
added to the workflow.
* :data:`LinkEvent.InputLinkRemoved` - when a output link is removed.
* :data:`LinkEvent.InputLinkStateChanged` - when the input link's
runtime state changes.
* :data:`LinkEvent.OutputLinkStateChanged` - when the output link's
runtime state changes.
* :data:`WorkflowEnvEvent.WorkflowEnvironmentChanged` - when the
workflow environment changes.
.. seealso:: :func:`.Scheme.add_link()`, :func:`Scheme.remove_link`,
:func:`.Scheme.runtime_env`
"""
#: A new QWidget was created and added by the manager.
widget_for_node_added = Signal(SchemeNode, QWidget)
#: A QWidget was removed, hidden and will be deleted when appropriate.
widget_for_node_removed = Signal(SchemeNode, QWidget)
class CreationPolicy(enum.Enum):
"""
Widget Creation Policy.
"""
#: Widgets are scheduled to be created from the event loop, or when
#: first accessed with `widget_for_node`
Normal = "Normal"
#: Widgets are created immediately when a node is added to the
#: workflow model.
Immediate = "Immediate"
#: Widgets are created only when first accessed with `widget_for_node`
#: (e.g. when activated in the view).
OnDemand = "OnDemand"
Normal = CreationPolicy.Normal
Immediate = CreationPolicy.Immediate
OnDemand = CreationPolicy.OnDemand
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__workflow = None # type: Optional[Scheme]
self.__creation_policy = WidgetManager.Normal
self.__float_widgets_on_top = False
self.__item_for_node = {} # type: Dict[SchemeNode, Item]
self.__item_for_widget = {} # type: Dict[QWidget, Item]
self.__init_queue = deque() # type: Deque[SchemeNode]
self.__init_timer = QTimer(self, singleShot=True)
self.__init_timer.timeout.connect(self.__process_init_queue)
self.__activation_monitor = ActivationMonitor(self)
self.__activation_counter = itertools.count()
self.__activation_monitor.activated.connect(self.__mark_activated)
def set_workflow(self, workflow):
# type: (Scheme) -> None
"""
Set the workflow.
"""
if workflow is self.__workflow:
return
if self.__workflow is not None:
# cleanup
for node in self.__workflow.nodes:
self.__remove_node(node)
self.__workflow.node_added.disconnect(self.__on_node_added)
self.__workflow.node_removed.disconnect(self.__on_node_removed)
self.__workflow.link_added.disconnect(self.__on_link_added)
self.__workflow.link_removed.disconnect(self.__on_link_removed)
self.__workflow.runtime_env_changed.disconnect(
self.__on_env_changed)
self.__workflow.removeEventFilter(self)
self.__workflow = workflow
workflow.node_added.connect(self.__on_node_added, Qt.UniqueConnection)
workflow.node_removed.connect(self.__on_node_removed,
Qt.UniqueConnection)
workflow.link_added.connect(self.__on_link_added, Qt.UniqueConnection)
workflow.link_removed.connect(self.__on_link_removed,
Qt.UniqueConnection)
workflow.runtime_env_changed.connect(self.__on_env_changed,
Qt.UniqueConnection)
workflow.installEventFilter(self)
for node in workflow.nodes:
self.__add_node(node)
def workflow(self):
# type: () -> Optional[Workflow]
return self.__workflow
scheme = workflow
set_scheme = set_workflow
def set_creation_policy(self, policy):
# type: (CreationPolicy) -> None
"""
Set the widget creation policy.
"""
if self.__creation_policy != policy:
self.__creation_policy = policy
if self.__creation_policy == WidgetManager.Immediate:
self.__init_timer.stop()
# create all
if self.__workflow is not None:
for node in self.__workflow.nodes:
self.ensure_created(node)
elif self.__creation_policy == WidgetManager.Normal:
if not self.__init_timer.isActive() and self.__init_queue:
self.__init_timer.start()
elif self.__creation_policy == WidgetManager.OnDemand:
self.__init_timer.stop()
else:
assert False
def creation_policy(self):
"""
Return the current widget creation policy.
"""
return self.__creation_policy
def create_widget_for_node(self, node):
# type: (SchemeNode) -> QWidget
"""
Create and initialize a widget for node.
This is an abstract method. Subclasses must reimplemented it.
"""
raise NotImplementedError()
def delete_widget_for_node(self, node, widget):
# type: (SchemeNode, QWidget) -> None
"""
Remove and delete widget for node.
This is an abstract method. Subclasses must reimplemented it.
"""
raise NotImplementedError()
def node_for_widget(self, widget):
# type: (QWidget) -> Optional[SchemeNode]
"""
Return the node for widget.
"""
item = self.__item_for_widget.get(widget)
if item is not None:
return item.node
else:
return None
def widget_for_node(self, node):
# type: (SchemeNode) -> Optional[QWidget]
"""
Return the widget for node.
"""
self.ensure_created(node)
item = self.__item_for_node.get(node)
return item.widget if item is not None else None
def __add_widget_for_node(self, node):
# type: (SchemeNode) -> None
item = self.__item_for_node.get(node)
if item is not None:
return
if self.__workflow is None:
return
if node not in self.__workflow.nodes:
return
if node in self.__init_queue:
self.__init_queue.remove(node)
item = Item(node, None, -1)
# Insert on the node -> item mapping.
self.__item_for_node[node] = item
log.debug("Creating widget for node %s", node)
try:
w = self.create_widget_for_node(node)
except Exception: # pylint: disable=broad-except
log.critical("", exc_info=True)
lines = traceback.format_exception(*sys.exc_info())
text = "".join(lines)
errorwidget = QLabel(textInteractionFlags=Qt.TextSelectableByMouse,
wordWrap=True,
objectName="widgetmanager-error-placeholder",
text="<pre>" + escape(text) + "</pre>")
item.errorwidget = errorwidget
node.set_state_message(UserMessage(text, UserMessage.Error, ""))
raise
else:
item.widget = w
self.__item_for_widget[w] = item
self.__set_float_on_top_flag(w)
if w.windowIcon().isNull():
desc = node.description
w.setWindowIcon(icon_loader.from_description(desc).get(desc.icon))
if not w.windowTitle():
w.setWindowTitle(node.title)
w.installEventFilter(self.__activation_monitor)
raise_canvas = QAction(
self.tr("Raise Canvas to Front"),
w,
objectName="action-canvas-raise-canvas",
toolTip=self.tr("Raise containing canvas workflow window"),
shortcut=QKeySequence(Qt.ControlModifier | Qt.Key_Up))
raise_canvas.triggered.connect(self.__on_activate_parent)
raise_descendants = QAction(
self.tr("Raise Descendants"),
w,
objectName="action-canvas-raise-descendants",
toolTip=self.tr("Raise all immediate descendants of this node"),
shortcut=QKeySequence(Qt.ControlModifier | Qt.ShiftModifier
| Qt.Key_Right))
raise_descendants.triggered.connect(
partial(self.__on_raise_descendants, node))
raise_ancestors = QAction(
self.tr("Raise Ancestors"),
w,
objectName="action-canvas-raise-ancestors",
toolTip=self.tr("Raise all immediate ancestors of this node"),
shortcut=QKeySequence(Qt.ControlModifier | Qt.ShiftModifier
| Qt.Key_Left))
raise_ancestors.triggered.connect(
partial(self.__on_raise_ancestors, node))
w.addActions([raise_canvas, raise_descendants, raise_ancestors])
# send all the post creation notification events
workflow = self.__workflow
assert workflow is not None
inputs = workflow.find_links(sink_node=node)
for link in inputs:
ev = LinkEvent(LinkEvent.InputLinkAdded, link)
QCoreApplication.sendEvent(w, ev)
outputs = workflow.find_links(source_node=node)
for link in outputs:
ev = LinkEvent(LinkEvent.OutputLinkAdded, link)
QCoreApplication.sendEvent(w, ev)
self.widget_for_node_added.emit(node, w)
def ensure_created(self, node):
# type: (SchemeNode) -> None
"""
Ensure that the widget for node is created.
"""
if self.__workflow is None:
return
if node not in self.__workflow.nodes:
return
item = self.__item_for_node.get(node)
if item is None:
self.__add_widget_for_node(node)
def __on_node_added(self, node):
# type: (SchemeNode) -> None
assert self.__workflow is not None
assert node in self.__workflow.nodes
assert node not in self.__item_for_node
self.__add_node(node)
def __add_node(self, node):
# type: (SchemeNode) -> None
# add node for tracking
node.installEventFilter(self)
if self.__creation_policy == WidgetManager.Immediate:
self.ensure_created(node)
else:
self.__init_queue.append(node)
if self.__creation_policy == WidgetManager.Normal:
self.__init_timer.start()
def __on_node_removed(self, node): # type: (SchemeNode) -> None
assert self.__workflow is not None
assert node not in self.__workflow.nodes
self.__remove_node(node)
def __remove_node(self, node): # type: (SchemeNode) -> None
# remove the node and its widget from tracking.
node.removeEventFilter(self)
if node in self.__init_queue:
self.__init_queue.remove(node)
item = self.__item_for_node.get(node)
if item is not None and item.widget is not None:
widget = item.widget
assert widget in self.__item_for_widget
del self.__item_for_widget[widget]
widget.removeEventFilter(self.__activation_monitor)
item.widget = None
self.widget_for_node_removed.emit(node, widget)
self.delete_widget_for_node(node, widget)
if item is not None:
del self.__item_for_node[node]
@Slot()
def __process_init_queue(self):
if self.__init_queue:
node = self.__init_queue.popleft()
assert self.__workflow is not None
assert node in self.__workflow.nodes
log.debug("__process_init_queue: '%s'", node.title)
try:
self.ensure_created(node)
finally:
if self.__init_queue:
self.__init_timer.start()
def __on_link_added(self, link): # type: (SchemeLink) -> None
assert self.__workflow is not None
assert link.source_node in self.__workflow.nodes
assert link.sink_node in self.__workflow.nodes
source = self.__item_for_node.get(link.source_node)
sink = self.__item_for_node.get(link.sink_node)
# notify the node gui of an added link
if source is not None and source.widget is not None:
ev = LinkEvent(LinkEvent.OutputLinkAdded, link)
QCoreApplication.sendEvent(source.widget, ev)
if sink is not None and sink.widget is not None:
ev = LinkEvent(LinkEvent.InputLinkAdded, link)
QCoreApplication.sendEvent(sink.widget, ev)
def __on_link_removed(self, link): # type: (SchemeLink) -> None
assert self.__workflow is not None
assert link.source_node in self.__workflow.nodes
assert link.sink_node in self.__workflow.nodes
source = self.__item_for_node.get(link.source_node)
sink = self.__item_for_node.get(link.sink_node)
# notify the node gui of an removed link
if source is not None and source.widget is not None:
ev = LinkEvent(LinkEvent.OutputLinkRemoved, link)
QCoreApplication.sendEvent(source.widget, ev)
if sink is not None and sink.widget is not None:
ev = LinkEvent(LinkEvent.InputLinkRemoved, link)
QCoreApplication.sendEvent(sink.widget, ev)
def __mark_activated(self, widget): # type: (QWidget) -> None
# Update the tracked stacking order for `widget`
item = self.__item_for_widget.get(widget)
if item is not None:
item.activation_order = next(self.__activation_counter)
def activate_widget_for_node(self, node, widget):
# type: (SchemeNode, QWidget) -> None
"""
Activate the widget for node (show and raise above other)
"""
if widget.windowState() == Qt.WindowMinimized:
widget.showNormal()
widget.setVisible(True)
widget.raise_()
widget.activateWindow()
def activate_window_group(self, group):
# type: (Scheme.WindowGroup) -> None
self.restore_window_state(group.state)
def raise_widgets_to_front(self):
"""
Raise all current visible widgets to the front.
The widgets will be stacked by activation order.
"""
workflow = self.__workflow
if workflow is None:
return
items = filter(
lambda item:
(item.widget.isVisible()
if item is not None and item.widget is not None else False),
map(self.__item_for_node.get, workflow.nodes))
self.__raise_and_activate(items)
def set_float_widgets_on_top(self, float_on_top):
"""
Set `Float Widgets on Top` flag on all widgets.
"""
self.__float_widgets_on_top = float_on_top
for item in self.__item_for_node.values():
if item.widget is not None:
self.__set_float_on_top_flag(item.widget)
def save_window_state(self):
# type: () -> List[Tuple[SchemeNode, bytes]]
"""
Save current open window arrangement.
"""
if self.__workflow is None:
return []
workflow = self.__workflow # type: Scheme
state = []
for node in workflow.nodes: # type: SchemeNode
item = self.__item_for_node.get(node, None)
if item is None:
continue
stackorder = item.activation_order
if item.widget is not None and not item.widget.isHidden():
data = self.save_widget_geometry(node, item.widget)
state.append((stackorder, node, data))
return [(node, data)
for _, node, data in sorted(state, key=lambda t: t[0])]
def restore_window_state(self, state):
# type: (List[Tuple[Node, bytes]]) -> None
"""
Restore the window state.
"""
assert self.__workflow is not None
workflow = self.__workflow # type: Scheme
visible = {node for node, _ in state}
# first hide all other widgets
for node in workflow.nodes:
if node not in visible:
# avoid creating widgets if not needed
item = self.__item_for_node.get(node, None)
if item is not None and item.widget is not None:
item.widget.hide()
allnodes = set(workflow.nodes)
# restore state for visible group; windows are stacked as they appear
# in the state list.
w = None
for node, node_state in filter(lambda t: t[0] in allnodes, state):
w = self.widget_for_node(node) # also create it if needed
if w is not None:
w.show()
self.restore_widget_geometry(node, w, node_state)
w.raise_()
self.__mark_activated(w)
# activate (give focus to) the last window
if w is not None:
w.activateWindow()
def save_widget_geometry(self, node, widget):
# type: (SchemeNode, QWidget) -> bytes
"""
Save and return the current geometry and state for node.
"""
return b''
def restore_widget_geometry(self, node, widget, state):
# type: (SchemeNode, QWidget, bytes) -> bool
"""
Restore the widget geometry and state for node.
Return True if the geometry was restored successfully.
The default implementation does nothing.
"""
return False
@Slot(SchemeNode)
def __on_raise_ancestors(self, node):
# type: (SchemeNode) -> None
"""
Raise all the ancestor widgets of `widget`.
"""
item = self.__item_for_node.get(node)
if item is not None:
scheme = self.scheme()
assert scheme is not None
ancestors = [
self.__item_for_node.get(p) for p in scheme.parents(item.node)
]
self.__raise_and_activate(filter(None, reversed(ancestors)))
@Slot(SchemeNode)
def __on_raise_descendants(self, node):
# type: (SchemeNode) -> None
"""
Raise all the descendants widgets of `widget`.
"""
item = self.__item_for_node.get(node)
if item is not None:
scheme = self.scheme()
assert scheme is not None
descendants = [
self.__item_for_node.get(p) for p in scheme.children(item.node)
]
self.__raise_and_activate(filter(None, reversed(descendants)))
def __raise_and_activate(self, items):
# type: (Iterable[Item]) -> None
"""Show and raise a set of widgets."""
# preserve the tracked stacking order
items = sorted(items, key=lambda item: item.activation_order)
w = None
for item in items:
if item.widget is not None:
w = item.widget
elif item.errorwidget is not None:
w = item.errorwidget
else:
continue
w.show()
w.raise_()
if w is not None:
# give focus to the last activated top window
w.activateWindow()
def __activate_widget_for_node(self, node): # type: (SchemeNode) -> None
# activate the widget for the node.
self.ensure_created(node)
item = self.__item_for_node.get(node)
if item is None:
return
if item.widget is not None:
self.activate_widget_for_node(node, item.widget)
elif item.errorwidget is not None:
item.errorwidget.show()
item.errorwidget.raise_()
item.errorwidget.activateWindow()
def __on_activate_parent(self):
event = WorkflowEvent(WorkflowEvent.ActivateParentRequest)
QCoreApplication.sendEvent(self.scheme(), event)
def eventFilter(self, recv, event):
# type: (QObject, QEvent) -> bool
if isinstance(recv, SchemeNode):
if event.type() == NodeEvent.NodeActivateRequest:
self.__activate_widget_for_node(recv)
self.__dispatch_events(recv, event)
return False
def __dispatch_events(self, node: Node, event: QEvent) -> None:
"""
Dispatch relevant workflow events to the GUI widget
"""
if event.type() in (
WorkflowEvent.InputLinkAdded,
WorkflowEvent.InputLinkRemoved,
WorkflowEvent.InputLinkStateChange,
WorkflowEvent.OutputLinkAdded,
WorkflowEvent.OutputLinkRemoved,
WorkflowEvent.OutputLinkStateChange,
WorkflowEvent.NodeStateChange,
WorkflowEvent.WorkflowEnvironmentChange,
):
item = self.__item_for_node.get(node)
if item is not None and item.widget is not None:
QCoreApplication.sendEvent(item.widget, event)
def __set_float_on_top_flag(self, widget):
# type: (QWidget) -> None
"""Set or unset widget's float on top flag"""
should_float_on_top = self.__float_widgets_on_top
float_on_top = bool(widget.windowFlags() & Qt.WindowStaysOnTopHint)
if float_on_top == should_float_on_top:
return
widget_was_visible = widget.isVisible()
if should_float_on_top:
widget.setWindowFlags(widget.windowFlags()
| Qt.WindowStaysOnTopHint)
else:
widget.setWindowFlags(widget.windowFlags()
& ~Qt.WindowStaysOnTopHint)
# Changing window flags hid the widget
if widget_was_visible:
widget.show()
def __on_env_changed(self, key, newvalue, oldvalue):
# Notify widgets of a runtime environment change
for item in self.__item_for_node.values():
if item.widget is not None:
ev = WorkflowEnvChanged(key, newvalue, oldvalue)
QCoreApplication.sendEvent(item.widget, ev)
def actions_for_context_menu(self, node):
# type: (SchemeNode) -> List[QAction]
"""
Return a list of extra actions that can be inserted into context
menu in the workflow editor.
Subclasses can reimplement this method to extend the default context
menu.
Parameters
----------
node: SchemeNode
The node for which the context menu is requested.
Return
------
actions: List[QAction]
Actions that are appended to the default menu.
"""
return []
class CanvasView(QGraphicsView):
"""Canvas View handles the zooming.
"""
def __init__(self, *args):
QGraphicsView.__init__(self, *args)
self.setAlignment(Qt.AlignTop | Qt.AlignLeft)
self.__backgroundIcon = QIcon()
self.__autoScroll = False
self.__autoScrollMargin = 16
self.__autoScrollTimer = QTimer(self)
self.__autoScrollTimer.timeout.connect(self.__autoScrollAdvance)
def setScene(self, scene):
QGraphicsView.setScene(self, scene)
self._ensureSceneRect(scene)
def _ensureSceneRect(self, scene):
r = scene.addRect(QRectF(0, 0, 400, 400))
scene.sceneRect()
scene.removeItem(r)
def setAutoScrollMargin(self, margin):
self.__autoScrollMargin = margin
def autoScrollMargin(self):
return self.__autoScrollMargin
def setAutoScroll(self, enable):
self.__autoScroll = enable
def autoScroll(self):
return self.__autoScroll
def mousePressEvent(self, event):
QGraphicsView.mousePressEvent(self, event)
def mouseMoveEvent(self, event):
if event.buttons() & Qt.LeftButton:
if not self.__autoScrollTimer.isActive() and \
self.__shouldAutoScroll(event.pos()):
self.__startAutoScroll()
QGraphicsView.mouseMoveEvent(self, event)
def mouseReleaseEvent(self, event):
if event.button() & Qt.LeftButton:
self.__stopAutoScroll()
return QGraphicsView.mouseReleaseEvent(self, event)
def __shouldAutoScroll(self, pos):
if self.__autoScroll:
margin = self.__autoScrollMargin
viewrect = self.contentsRect()
rect = viewrect.adjusted(margin, margin, -margin, -margin)
# only do auto scroll when on the viewport's margins
return not rect.contains(pos) and viewrect.contains(pos)
else:
return False
def __startAutoScroll(self):
self.__autoScrollTimer.start(10)
log.debug("Auto scroll timer started")
def __stopAutoScroll(self):
if self.__autoScrollTimer.isActive():
self.__autoScrollTimer.stop()
log.debug("Auto scroll timer stopped")
def __autoScrollAdvance(self):
"""Advance the auto scroll
"""
pos = QCursor.pos()
pos = self.mapFromGlobal(pos)
margin = self.__autoScrollMargin
vvalue = self.verticalScrollBar().value()
hvalue = self.horizontalScrollBar().value()
vrect = QRect(0, 0, self.width(), self.height())
# What should be the speed
advance = 10
# We only do auto scroll if the mouse is inside the view.
if vrect.contains(pos):
if pos.x() < vrect.left() + margin:
self.horizontalScrollBar().setValue(hvalue - advance)
if pos.y() < vrect.top() + margin:
self.verticalScrollBar().setValue(vvalue - advance)
if pos.x() > vrect.right() - margin:
self.horizontalScrollBar().setValue(hvalue + advance)
if pos.y() > vrect.bottom() - margin:
self.verticalScrollBar().setValue(vvalue + advance)
if self.verticalScrollBar().value() == vvalue and \
self.horizontalScrollBar().value() == hvalue:
self.__stopAutoScroll()
else:
self.__stopAutoScroll()
log.debug("Auto scroll advance")
def setBackgroundIcon(self, icon):
if not isinstance(icon, QIcon):
raise TypeError("A QIcon expected.")
if self.__backgroundIcon != icon:
self.__backgroundIcon = icon
self.viewport().update()
def backgroundIcon(self):
return QIcon(self.__backgroundIcon)
def drawBackground(self, painter, rect):
QGraphicsView.drawBackground(self, painter, rect)
if not self.__backgroundIcon.isNull():
painter.setClipRect(rect)
vrect = QRect(QPoint(0, 0), self.viewport().size())
vrect = self.mapToScene(vrect).boundingRect()
pm = self.__backgroundIcon.pixmap(
vrect.size().toSize().boundedTo(QSize(200, 200))
)
pmrect = QRect(QPoint(0, 0), pm.size())
pmrect.moveCenter(vrect.center().toPoint())
if rect.toRect().intersects(pmrect):
painter.drawPixmap(pmrect, pm)
class OWLouvainClustering(widget.OWWidget):
name = 'Louvain Clustering'
description = 'Detects communities in a network of nearest neighbors.'
icon = 'icons/LouvainClustering.svg'
priority = 2110
want_main_area = False
settingsHandler = DomainContextHandler()
class Inputs:
data = Input('Data', Table, default=True)
if Graph is not None:
class Outputs:
annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME,
Table,
default=True)
graph = Output('Network', Graph)
else:
class Outputs:
annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME,
Table,
default=True)
apply_pca = ContextSetting(True)
pca_components = ContextSetting(_DEFAULT_PCA_COMPONENTS)
metric_idx = ContextSetting(0)
k_neighbors = ContextSetting(_DEFAULT_K_NEIGHBORS)
resolution = ContextSetting(1.)
auto_commit = Setting(False)
class Information(widget.OWWidget.Information):
modified = Msg("Press commit to recompute clusters and send new data")
class Error(widget.OWWidget.Error):
empty_dataset = Msg('No features in data')
general_error = Msg('Error occured during clustering\n{}')
def __init__(self):
super().__init__()
self.data = None # type: Optional[Table]
self.preprocessed_data = None # type: Optional[Table]
self.pca_projection = None # type: Optional[Table]
self.graph = None # type: Optional[nx.Graph]
self.partition = None # type: Optional[np.array]
# Use a executor with a single worker, to limit CPU overcommitment for
# cancelled tasks. The method does not have a fine cancellation
# granularity so we assure that there are not N - 1 jobs executing
# for no reason only to be thrown away. It would be better to use the
# global pool but implement a limit on jobs from this source.
self.__executor = futures.ThreadPoolExecutor(max_workers=1)
self.__task = None # type: Optional[TaskState]
self.__invalidated = False
# coalescing commit timer
self.__commit_timer = QTimer(self, singleShot=True)
self.__commit_timer.timeout.connect(self.commit)
pca_box = gui.vBox(self.controlArea, 'PCA Preprocessing')
self.apply_pca_cbx = gui.checkBox(
pca_box,
self,
'apply_pca',
label='Apply PCA preprocessing',
callback=self._invalidate_graph,
) # type: QCheckBox
self.pca_components_slider = gui.hSlider(
pca_box,
self,
'pca_components',
label='Components: ',
minValue=2,
maxValue=_MAX_PCA_COMPONENTS,
callback=self._invalidate_pca_projection,
tracking=False) # type: QSlider
graph_box = gui.vBox(self.controlArea, 'Graph parameters')
self.metric_combo = gui.comboBox(
graph_box,
self,
'metric_idx',
label='Distance metric',
items=[m[0] for m in METRICS],
callback=self._invalidate_graph,
orientation=Qt.Horizontal,
) # type: gui.OrangeComboBox
self.k_neighbors_spin = gui.spin(
graph_box,
self,
'k_neighbors',
minv=1,
maxv=_MAX_K_NEIGBOURS,
label='k neighbors',
controlWidth=80,
alignment=Qt.AlignRight,
callback=self._invalidate_graph,
) # type: gui.SpinBoxWFocusOut
self.resolution_spin = gui.hSlider(
graph_box,
self,
'resolution',
minValue=0,
maxValue=5.,
step=1e-1,
label='Resolution',
intOnly=False,
labelFormat='%.1f',
callback=self._invalidate_partition,
tracking=False,
) # type: QSlider
self.resolution_spin.parent().setToolTip(
'The resolution parameter affects the number of clusters to find. '
'Smaller values tend to produce more clusters and larger values '
'retrieve less clusters.')
self.apply_button = gui.auto_commit(
self.controlArea,
self,
'auto_commit',
'Apply',
box=None,
commit=lambda: self.commit(),
callback=lambda: self._on_auto_commit_changed(),
) # type: QWidget
def _invalidate_pca_projection(self):
self.pca_projection = None
if not self.apply_pca:
return
self._invalidate_graph()
self._set_modified(True)
def _invalidate_graph(self):
self.graph = None
self._invalidate_partition()
self._set_modified(True)
def _invalidate_partition(self):
self.partition = None
self._invalidate_output()
self.Information.modified()
self._set_modified(True)
def _invalidate_output(self):
self.__invalidated = True
if self.__task is not None:
self.__cancel_task(wait=False)
if self.auto_commit:
self.__commit_timer.start()
else:
self.__set_state_ready()
def _set_modified(self, state):
"""
Mark the widget (GUI) as containing modified state.
"""
if self.data is None:
# does not apply when we have no data
state = False
elif self.auto_commit:
# does not apply when auto commit is on
state = False
self.Information.modified(shown=state)
def _on_auto_commit_changed(self):
if self.auto_commit and self.__invalidated:
self.commit()
def cancel(self):
"""Cancel any running jobs."""
self.__cancel_task(wait=False)
self.__set_state_ready()
def commit(self):
self.__commit_timer.stop()
self.__invalidated = False
self._set_modified(False)
self.Error.clear()
# Cancel current running task
self.__cancel_task(wait=False)
if self.data is None:
self.__set_state_ready()
return
# Make sure the dataset is ok
if len(self.data.domain.attributes) < 1:
self.Error.empty_dataset()
self.__set_state_ready()
return
if self.partition is not None:
self.__set_state_ready()
self._send_data()
return
# Preprocess the dataset
if self.preprocessed_data is None:
louvain = Louvain(random_state=0)
self.preprocessed_data = louvain.preprocess(self.data)
state = TaskState(self)
# Prepare/assemble the task(s) to run; reuse partial results
if self.apply_pca:
if self.pca_projection is not None:
data = self.pca_projection
pca_components = None
else:
data = self.preprocessed_data
pca_components = self.pca_components
else:
data = self.preprocessed_data
pca_components = None
if self.graph is not None:
# run on graph only; no need to do PCA and k-nn search ...
graph = self.graph
k_neighbors = metric = None
else:
k_neighbors, metric = self.k_neighbors, METRICS[self.metric_idx][1]
graph = None
if graph is None:
task = partial(run_on_data,
data,
pca_components=pca_components,
k_neighbors=k_neighbors,
metric=metric,
resolution=self.resolution,
state=state)
else:
task = partial(run_on_graph,
graph,
resolution=self.resolution,
state=state)
self.__set_state_busy()
self.__start_task(task, state)
@Slot(object)
def __set_partial_results(self, result):
# type: (Tuple[str, Any]) -> None
which, res = result
if which == "pca_projection":
assert isinstance(res, Table) and len(res) == len(self.data)
self.pca_projection = res
elif which == "graph":
assert isinstance(res, nx.Graph)
self.graph = res
elif which == "partition":
assert isinstance(res, np.ndarray)
self.partition = res
else:
assert False, which
@Slot(object)
def __on_done(self, future):
# type: (Future['Results']) -> None
assert future.done()
assert self.__task is not None
assert self.__task.future is future
assert self.__task.watcher.future() is future
self.__task, task = None, self.__task
task.deleteLater()
self.__set_state_ready()
try:
result = future.result()
except Exception as err: # pylint: disable=broad-except
self.Error.general_error(str(err), exc_info=True)
else:
self.__set_results(result)
@Slot(str)
def setStatusMessage(self, text):
super().setStatusMessage(text)
@Slot(float)
def progressBarSet(self, value, *a, **kw):
super().progressBarSet(value, *a, **kw)
def __set_state_ready(self):
self.progressBarFinished()
self.setBlocking(False)
self.setStatusMessage("")
def __set_state_busy(self):
self.progressBarInit()
self.setBlocking(True)
def __start_task(self, task, state):
# type: (Callable[[], Any], TaskState) -> None
assert self.__task is None
state.status_changed.connect(self.setStatusMessage)
state.progress_changed.connect(self.progressBarSet)
state.partial_result_ready.connect(self.__set_partial_results)
state.watcher.done.connect(self.__on_done)
state.start(self.__executor, task)
state.setParent(self)
self.__task = state
def __cancel_task(self, wait=True):
# Cancel and dispose of the current task
if self.__task is not None:
state, self.__task = self.__task, None
state.cancel()
state.partial_result_ready.disconnect(self.__set_partial_results)
state.status_changed.disconnect(self.setStatusMessage)
state.progress_changed.disconnect(self.progressBarSet)
state.watcher.done.disconnect(self.__on_done)
if wait:
futures.wait([state.future])
state.deleteLater()
else:
w = FutureWatcher(state.future, parent=state)
w.done.connect(state.deleteLater)
def __set_results(self, results):
# type: ('Results') -> None
# NOTE: All of these have already been set by __set_partial_results,
# we double check that they are aliases
if results.pca_projection is not None:
assert self.pca_components == results.pca_components
assert self.pca_projection is results.pca_projection
self.pca_projection = results.pca_projection
if results.graph is not None:
assert results.metric == METRICS[self.metric_idx][1]
assert results.k_neighbors == self.k_neighbors
assert self.graph is results.graph
self.graph = results.graph
if results.partition is not None:
assert results.resolution == self.resolution
assert self.partition is results.partition
self.partition = results.partition
self._send_data()
def _send_data(self):
if self.partition is None or self.data is None:
return
domain = self.data.domain
# Compute the frequency of each cluster index
counts = np.bincount(self.partition)
indices = np.argsort(counts)[::-1]
index_map = {n: o for n, o in zip(indices, range(len(indices)))}
new_partition = list(map(index_map.get, self.partition))
cluster_var = DiscreteVariable(
get_unique_names(domain, 'Cluster'),
values=[
'C%d' % (i + 1) for i, _ in enumerate(np.unique(new_partition))
])
new_domain = add_columns(domain, metas=[cluster_var])
new_table = self.data.transform(new_domain)
new_table.get_column_view(cluster_var)[0][:] = new_partition
self.Outputs.annotated_data.send(new_table)
if Graph is not None:
graph = Graph(self.graph)
graph.set_items(new_table)
self.Outputs.graph.send(graph)
@Inputs.data
def set_data(self, data):
self.closeContext()
self.Error.clear()
prev_data, self.data = self.data, data
self.openContext(self.data)
# If X hasn't changed, there's no reason to recompute clusters
if prev_data and self.data and np.array_equal(self.data.X,
prev_data.X):
if self.auto_commit:
self._send_data()
return
# Clear the outputs
self.Outputs.annotated_data.send(None)
if Graph is not None:
self.Outputs.graph.send(None)
# Clear internal state
self.clear()
self._invalidate_pca_projection()
if self.data is None:
return
# Can't have more PCA components than the number of attributes
n_attrs = len(data.domain.attributes)
self.pca_components_slider.setMaximum(min(_MAX_PCA_COMPONENTS,
n_attrs))
self.pca_components_slider.setValue(
min(_DEFAULT_PCA_COMPONENTS, n_attrs))
# Can't have more k neighbors than there are data points
self.k_neighbors_spin.setMaximum(min(_MAX_K_NEIGBOURS, len(data) - 1))
self.k_neighbors_spin.setValue(min(_DEFAULT_K_NEIGHBORS,
len(data) - 1))
self.commit()
def clear(self):
self.__cancel_task(wait=False)
self.preprocessed_data = None
self.pca_projection = None
self.graph = None
self.partition = None
self.Error.clear()
self.Information.modified.clear()
def onDeleteWidget(self):
self.__cancel_task(wait=True)
self.__executor.shutdown(True)
self.clear()
self.data = None
super().onDeleteWidget()
def send_report(self):
pca = report.bool_str(self.apply_pca)
if self.apply_pca:
pca += report.plural(', {number} component{s}',
self.pca_components)
self.report_items((
('PCA preprocessing', pca),
('Metric', METRICS[self.metric_idx][0]),
('k neighbors', self.k_neighbors),
('Resolution', self.resolution),
))
class OWPCA(widget.OWWidget):
name = "PCA"
description = "Principal component analysis with a scree-diagram."
icon = "icons/PCA.svg"
priority = 3050
inputs = [("Data", Table, "set_data")]
outputs = [("Transformed data", Table),
("Components", Table),
("PCA", PCA)]
ncomponents = settings.Setting(2)
variance_covered = settings.Setting(100)
batch_size = settings.Setting(100)
address = settings.Setting('')
auto_update = settings.Setting(True)
auto_commit = settings.Setting(True)
normalize = settings.Setting(True)
maxp = settings.Setting(20)
axis_labels = settings.Setting(10)
graph_name = "plot.plotItem"
def __init__(self):
super().__init__()
self.data = None
self._pca = None
self._transformed = None
self._variance_ratio = None
self._cumulative = None
self._line = False
self._pca_projector = PCA()
self._pca_projector.component = self.ncomponents
self._pca_preprocessors = PCA.preprocessors
# Components Selection
box = gui.vBox(self.controlArea, "Components Selection")
form = QFormLayout()
box.layout().addLayout(form)
self.components_spin = gui.spin(
box, self, "ncomponents", 0, 1000,
callback=self._update_selection_component_spin,
keyboardTracking=False
)
self.components_spin.setSpecialValueText("All")
self.variance_spin = gui.spin(
box, self, "variance_covered", 1, 100,
callback=self._update_selection_variance_spin,
keyboardTracking=False
)
self.variance_spin.setSuffix("%")
form.addRow("Components:", self.components_spin)
form.addRow("Variance covered:", self.variance_spin)
# Incremental learning
self.sampling_box = gui.vBox(self.controlArea, "Incremental learning")
self.addresstext = QLineEdit(box)
self.addresstext.setPlaceholderText('Remote server')
if self.address:
self.addresstext.setText(self.address)
self.sampling_box.layout().addWidget(self.addresstext)
form = QFormLayout()
self.sampling_box.layout().addLayout(form)
self.batch_spin = gui.spin(
self.sampling_box, self, "batch_size", 50, 100000, step=50,
keyboardTracking=False)
form.addRow("Batch size ~ ", self.batch_spin)
self.start_button = gui.button(
self.sampling_box, self, "Start remote computation",
callback=self.start, autoDefault=False,
tooltip="Start/abort computation on the server")
self.start_button.setEnabled(False)
gui.checkBox(self.sampling_box, self, "auto_update",
"Periodically fetch model", callback=self.update_model)
self.__timer = QTimer(self, interval=2000)
self.__timer.timeout.connect(self.get_model)
self.sampling_box.setVisible(remotely)
# Options
self.options_box = gui.vBox(self.controlArea, "Options")
gui.checkBox(self.options_box, self, "normalize", "Normalize data",
callback=self._update_normalize)
self.maxp_spin = gui.spin(
self.options_box, self, "maxp", 1, 100,
label="Show only first", callback=self._setup_plot,
keyboardTracking=False
)
self.controlArea.layout().addStretch()
gui.auto_commit(self.controlArea, self, "auto_commit", "Apply",
checkbox_label="Apply automatically")
self.plot = pg.PlotWidget(background="w")
axis = self.plot.getAxis("bottom")
axis.setLabel("Principal Components")
axis = self.plot.getAxis("left")
axis.setLabel("Proportion of variance")
self.plot_horlabels = []
self.plot_horlines = []
self.plot.getViewBox().setMenuEnabled(False)
self.plot.getViewBox().setMouseEnabled(False, False)
self.plot.showGrid(True, True, alpha=0.5)
self.plot.setRange(xRange=(0.0, 1.0), yRange=(0.0, 1.0))
self.mainArea.layout().addWidget(self.plot)
self._update_normalize()
def update_model(self):
self.get_model()
if self.auto_update and self.rpca and not self.rpca.ready():
self.__timer.start(2000)
else:
self.__timer.stop()
def start(self):
if 'Abort' in self.start_button.text():
self.rpca.abort()
self.__timer.stop()
self.start_button.setText("Start remote computation")
else:
self.address = self.addresstext.text()
with remote.server(self.address):
from Orange.projection.pca import RemotePCA
maxiter = (1e5 + self.data.approx_len()) / self.batch_size * 3
self.rpca = RemotePCA(self.data, self.batch_size, int(maxiter))
self.update_model()
self.start_button.setText("Abort remote computation")
def set_data(self, data):
self.information()
if isinstance(data, SqlTable):
if data.approx_len() < AUTO_DL_LIMIT:
data = Table(data)
elif not remotely:
self.information("Data has been sampled")
data_sample = data.sample_time(1, no_cache=True)
data_sample.download_data(2000, partial=True)
data = Table(data_sample)
self.data = data
self.fit()
def fit(self):
self.clear()
self.start_button.setEnabled(False)
if self.data is None:
return
data = self.data
self._transformed = None
if isinstance(data, SqlTable): # data was big and remote available
self.sampling_box.setVisible(True)
self.start_button.setText("Start remote computation")
self.start_button.setEnabled(True)
else:
self.sampling_box.setVisible(False)
pca = self._pca_projector(data)
variance_ratio = pca.explained_variance_ratio_
cumulative = numpy.cumsum(variance_ratio)
self.components_spin.setRange(0, len(cumulative))
self._pca = pca
self._variance_ratio = variance_ratio
self._cumulative = cumulative
self._setup_plot()
self.unconditional_commit()
def clear(self):
self._pca = None
self._transformed = None
self._variance_ratio = None
self._cumulative = None
self._line = None
self.plot_horlabels = []
self.plot_horlines = []
self.plot.clear()
def get_model(self):
if self.rpca is None:
return
if self.rpca.ready():
self.__timer.stop()
self.start_button.setText("Restart (finished)")
self._pca = self.rpca.get_state()
if self._pca is None:
return
self._variance_ratio = self._pca.explained_variance_ratio_
self._cumulative = numpy.cumsum(self._variance_ratio)
self._setup_plot()
self._transformed = None
self.commit()
def _setup_plot(self):
self.plot.clear()
explained_ratio = self._variance_ratio
explained = self._cumulative
p = min(len(self._variance_ratio), self.maxp)
self.plot.plot(numpy.arange(p), explained_ratio[:p],
pen=pg.mkPen(QColor(Qt.red), width=2),
antialias=True,
name="Variance")
self.plot.plot(numpy.arange(p), explained[:p],
pen=pg.mkPen(QColor(Qt.darkYellow), width=2),
antialias=True,
name="Cumulative Variance")
cutpos = self._nselected_components() - 1
self._line = pg.InfiniteLine(
angle=90, pos=cutpos, movable=True, bounds=(0, p - 1))
self._line.setCursor(Qt.SizeHorCursor)
self._line.setPen(pg.mkPen(QColor(Qt.black), width=2))
self._line.sigPositionChanged.connect(self._on_cut_changed)
self.plot.addItem(self._line)
self.plot_horlines = (
pg.PlotCurveItem(pen=pg.mkPen(QColor(Qt.blue), style=Qt.DashLine)),
pg.PlotCurveItem(pen=pg.mkPen(QColor(Qt.blue), style=Qt.DashLine)))
self.plot_horlabels = (
pg.TextItem(color=QColor(Qt.black), anchor=(1, 0)),
pg.TextItem(color=QColor(Qt.black), anchor=(1, 1)))
for item in self.plot_horlabels + self.plot_horlines:
self.plot.addItem(item)
self._set_horline_pos()
self.plot.setRange(xRange=(0.0, p - 1), yRange=(0.0, 1.0))
self._update_axis()
def _set_horline_pos(self):
cutidx = self.ncomponents - 1
for line, label, curve in zip(self.plot_horlines, self.plot_horlabels,
(self._variance_ratio, self._cumulative)):
y = curve[cutidx]
line.setData([-1, cutidx], 2 * [y])
label.setPos(cutidx, y)
label.setPlainText("{:.3f}".format(y))
def _on_cut_changed(self, line):
# cut changed by means of a cut line over the scree plot.
value = int(round(line.value()))
self._line.setValue(value)
current = self._nselected_components()
components = value + 1
if not (self.ncomponents == 0 and
components == len(self._variance_ratio)):
self.ncomponents = components
self._set_horline_pos()
if self._pca is not None:
self.variance_covered = self._cumulative[components - 1] * 100
if current != self._nselected_components():
self._invalidate_selection()
def _update_selection_component_spin(self):
# cut changed by "ncomponents" spin.
if self._pca is None:
self._invalidate_selection()
return
if self.ncomponents == 0:
# Special "All" value
cut = len(self._variance_ratio)
else:
cut = self.ncomponents
self.variance_covered = self._cumulative[cut - 1] * 100
if numpy.floor(self._line.value()) + 1 != cut:
self._line.setValue(cut - 1)
self._invalidate_selection()
def _update_selection_variance_spin(self):
# cut changed by "max variance" spin.
if self._pca is None:
return
cut = numpy.searchsorted(self._cumulative,
self.variance_covered / 100.0) + 1
cut = min(cut, len(self._cumulative))
self.ncomponents = cut
if numpy.floor(self._line.value()) + 1 != cut:
self._line.setValue(cut - 1)
self._invalidate_selection()
def _update_normalize(self):
if self.normalize:
pp = self._pca_preprocessors + [Normalize()]
else:
pp = self._pca_preprocessors
self._pca_projector.preprocessors = pp
self.fit()
if self.data is None:
self._invalidate_selection()
def _nselected_components(self):
"""Return the number of selected components."""
if self._pca is None:
return 0
if self.ncomponents == 0:
# Special "All" value
max_comp = len(self._variance_ratio)
else:
max_comp = self.ncomponents
var_max = self._cumulative[max_comp - 1]
if var_max != numpy.floor(self.variance_covered / 100.0):
cut = max_comp
self.variance_covered = var_max * 100
else:
self.ncomponents = cut = numpy.searchsorted(
self._cumulative, self.variance_covered / 100.0) + 1
return cut
def _invalidate_selection(self):
self.commit()
def _update_axis(self):
p = min(len(self._variance_ratio), self.maxp)
axis = self.plot.getAxis("bottom")
d = max((p-1)//(self.axis_labels-1), 1)
axis.setTicks([[(i, str(i+1)) for i in range(0, p, d)]])
def commit(self):
transformed = components = None
if self._pca is not None:
if self._transformed is None:
# Compute the full transform (all components) only once.
self._transformed = self._pca(self.data)
transformed = self._transformed
domain = Domain(
transformed.domain.attributes[:self.ncomponents],
self.data.domain.class_vars,
self.data.domain.metas
)
transformed = transformed.from_table(domain, transformed)
dom = Domain(self._pca.orig_domain.attributes,
metas=[StringVariable(name='component')])
metas = numpy.array([['PC{}'.format(i + 1)
for i in range(self.ncomponents)]],
dtype=object).T
components = Table(dom, self._pca.components_[:self.ncomponents],
metas=metas)
components.name = 'components'
self._pca_projector.component = self.ncomponents
self.send("Transformed data", transformed)
self.send("Components", components)
self.send("PCA", self._pca_projector)
def send_report(self):
if self.data is None:
return
self.report_items((
("Selected components", self.ncomponents),
("Explained variance", "{:.3f} %".format(self.variance_covered))
))
self.report_plot()
