def test_methodinvoke(self):
executor = ThreadExecutor()
state = [None, None]
class StateSetter(QObject):
@Slot(object)
def set_state(self, value):
state[0] = value
state[1] = QThread.currentThread()
def func(callback):
callback(QThread.currentThread())
obj = StateSetter()
f1 = executor.submit(func, methodinvoke(obj, "set_state", (object,)))
f1.result()
# So invoked method can be called
QCoreApplication.processEvents()
self.assertIs(state[1], QThread.currentThread(),
"set_state was called from the wrong thread")
self.assertIsNot(state[0], QThread.currentThread(),
"set_state was invoked in the main thread")
executor.shutdown(wait=True)
def test_task(self):
results = []
task = Task(function=QThread.currentThread)
task.resultReady.connect(results.append)
task.start()
self.app.processEvents()
self.assertSequenceEqual(results, [QThread.currentThread()])
thread = QThread()
thread.start()
try:
task = Task(function=QThread.currentThread)
task.moveToThread(thread)
self.assertIsNot(task.thread(), QThread.currentThread())
self.assertIs(task.thread(), thread)
results = Future()
def record(value):
# record the result value and the calling thread
results.set_result((QThread.currentThread(), value))
task.resultReady.connect(record, Qt.DirectConnection)
task.start()
f = task.future()
emit_thread, thread_ = results.result(3)
self.assertIs(f.result(3), thread)
self.assertIs(emit_thread, thread)
self.assertIs(thread_, thread)
finally:
thread.quit()
thread.wait()
class X11EventPoller(QObject):
keyPressed = pyqtSignal(object, object)
def __init__(self):
QObject.__init__(self)
self._display = Display()
self._thread = QThread()
self.moveToThread(self._thread)
self._thread.start()
def start(self):
QTimer.singleShot(0, self.run)
def run(self):
ctx = self._display.record_create_context(0,
[record.CurrentClients],
[{
'core_requests': (0, 0),
'core_replies': (0, 0),
'ext_requests': (0, 0, 0, 0),
'ext_replies': (0, 0, 0, 0),
'delivered_events': (0, 0),
'device_events': (X.KeyPress, X.KeyRelease),
'errors': (0, 0),
'client_started': False,
'client_died': False,
}])
self._display.record_enable_context(ctx, self._record_callback)
self._display.record_free_context(ctx)
def _record_callback(self, reply):
QApplication.processEvents()
if reply.category != record.FromServer:
return
if reply.client_swapped:
# received swapped protocol data, cowardly ignored
return
if not len(reply.data) or reply.data[0] < 2:
# not an event
return
data = reply.data
while len(data):
event, data = rq.EventField(None).parse_binary_value(data, self._display.display, None, None)
self.keyPressed.emit(event, data)
def destroy(self):
# self._thread.terminate()
self._thread.wait()
def setBioMartDatasets(self, datasets):
assert(QThread.currentThread() is self.thread())
self.setEnabled(True)
self.datasets = [data for data in datasets if
getattr(data, "visible", "0") != "0"]
self.datasetsCombo.clear()
self.datasetsCombo.addItems([data.displayName for data in self.datasets])
def __emitpending(self, index, future):
# type: (int, Future) -> None
assert QThread.currentThread() is self.thread()
assert self.__futures[index] is future
assert future.done()
assert self.__countdone < len(self.__futures)
self.__futures[index] = None
self.__countdone += 1
if future.cancelled():
self.cancelledAt.emit(index, future)
self.doneAt.emit(index, future)
elif future.done():
self.finishedAt.emit(index, future)
self.doneAt.emit(index, future)
if future.exception():
self.exceptionReadyAt.emit(index, future.exception())
else:
self.resultReadyAt.emit(index, future.result())
else:
assert False
self.progressChanged.emit(self.__countdone, len(self.__futures))
if self.__countdone == len(self.__futures):
self.doneAll.emit()
def __accepted(self):
steps = self.addonwidget.itemState()
if steps:
# Move all uninstall steps to the front
steps = sorted(
steps, key=lambda step: 0 if step[0] == Uninstall else 1
)
self.__installer = Installer(steps=steps)
self.__thread = QThread(self)
self.__thread.start()
self.__installer.moveToThread(self.__thread)
self.__installer.finished.connect(self.__on_installer_finished)
self.__installer.error.connect(self.__on_installer_error)
progress = self.progressDialog()
self.__installer.installStatusChanged.connect(progress.setLabelText)
progress.show()
progress.setLabelText("Installing")
self.__installer.start()
else:
self.accept()
def _task_finished(self, f):
"""
Parameters:
----------
f: conncurent.futures.Future
future instance holding the result of learner evaluation
"""
assert self.thread() is QThread.currentThread()
assert self._task is not None
assert self._task.future is f
assert f.done()
self._task = None
if not self.was_canceled:
self.cancel_button.setDisabled(True)
try:
results = f.result()
except Exception as ex:
log = logging.getLogger()
log.exception(__name__, exc_info=True)
self.error("Exception occured during evaluation: {!r}".format(ex))
for key in self.results.keys():
self.results[key] = None
else:
self.update_view(results[1])
self.progressBarFinished(processEvents=False)
def run(self):
"""
Reimplemented from `QRunnable.run`
"""
self.eventLoop = QEventLoop()
self.eventLoop.processEvents()
# Move the task to the current thread so it's events, signals, slots
# are triggered from this thread.
assert self.task.thread() is _TaskDepotThread.instance()
QMetaObject.invokeMethod(
self.task.thread(), "transfer", Qt.BlockingQueuedConnection,
Q_ARG(object, self.task),
Q_ARG(object, QThread.currentThread())
)
self.eventLoop.processEvents()
# Schedule task.run from the event loop.
self.task.start()
# Quit the loop and exit when task finishes or is cancelled.
self.task.finished.connect(self.eventLoop.quit)
self.task.cancelled.connect(self.eventLoop.quit)
self.eventLoop.exec_()
def initialize(self):
"""
Clear and initialize the dialog.
This method must be called by the widget when the data is reset,
e.g. from `set_data` handler.
"""
if self._thread is not None and self._thread.isRunning():
self.keep_running = False
self._thread.quit()
self._thread.wait()
self.keep_running = False
self.scheduled_call = None
self.saved_state = None
self.saved_progress = 0
self.update_timer.stop()
self.progressBarFinished()
self.scores = []
self._update_model() # empty queue
self.rank_model.clear()
self.button.setText("Start")
self.button.setEnabled(self.check_preconditions())
self._thread = QThread(self)
self._worker = Worker(self)
self._worker.moveToThread(self._thread)
self._worker.stopped.connect(self._thread.quit)
self._worker.stopped.connect(self._select_first_if_none)
self._worker.stopped.connect(self._stopped)
self._worker.done.connect(self._done)
self._thread.started.connect(self._worker.do_work)
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
try:
corpus, errors = task.future.result()
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
corpus = None
errors = []
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if corpus:
self.n_text_data = len(corpus)
self.n_text_categories = len(corpus.domain.class_var.values)\
if corpus.domain.class_var else 0
self.corpus = corpus
self.n_skipped = len(errors)
if len(errors):
self.Warning.read_error("Some files" if len(errors) > 1 else "One file")
self.__setRuntimeState(state)
self.commit()
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
try:
data, n_skipped = task.future.result()
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
data = None
n_skipped = 0
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if data:
self._n_image_data = len(data)
self._n_image_categories = len(data.domain.class_var.values)\
if data.domain.class_var else 0
else:
self._n_image_data, self._n_image_categories = 0, 0
self.data = data
self._n_skipped = n_skipped
self.__setRuntimeState(state)
self.commit()
def _task_finished(self, f):
"""
Parameters
----------
f : Future
The future instance holding the built model
"""
assert self.thread() is QThread.currentThread()
assert self._task is not None
assert self._task.future is f
assert f.done()
self._task.deleteLater()
self._task = None
self.setBlocking(False)
self.progressBarFinished()
try:
self.model = f.result()
except Exception as ex: # pylint: disable=broad-except
# Log the exception with a traceback
log = logging.getLogger()
log.exception(__name__, exc_info=True)
self.model = None
self.show_fitting_failed(ex)
else:
self.model.name = self.learner_name
self.model.instances = self.data
self.model.skl_model.orange_callback = None # remove unpicklable callback
self.Outputs.model.send(self.model)
def __commit_complete(self, f):
# complete the commit operation after the required file has been
# downloaded
assert QThread.currentThread() is self.thread()
assert self.__awaiting_state is not None
assert self.__awaiting_state.future is f
if self.isBlocking():
self.progressBarFinished(processEvents=None)
self.setBlocking(False)
self.setStatusMessage("")
self.__awaiting_state = None
try:
path = f.result()
except Exception as ex:
log.exception("Error:")
self.error(format_exception(ex))
path = None
self.__update_cached_state()
if path is not None:
data = Orange.data.Table(path)
else:
data = None
self.Outputs.data.send(data)
def __onReportProgress(self, arg):
# report on scan progress from a worker thread
# arg must be a namespace(count: int, lastpath: str)
assert QThread.currentThread() is self.thread()
if self.__state == State.Processing:
self.pathlabel.setText(prettifypath(arg.lastpath))
self.progress_widget.setValue(arg.progress)
self.progress_widget.setValue(100 * arg.progress)
def _handleException(self, exception):
assert(QThread.currentThread() is self.thread())
print("Task failed with:", exception, file=sys.stderr)
import logging
log = logging.getLogger(__name__)
log.exception("Error:", exc_info=exception)
self.error(0, str(exception))
self.setEnabled(True)
def transfer(self, obj, thread):
"""
Transfer `obj` (:class:`QObject`) instance from this thread to the
target `thread` (a :class:`QThread`).
"""
assert obj.thread() is self
assert QThread.currentThread() is self
obj.moveToThread(thread)
def _stateChanged(self, future, state):
"""
The `future` state has changed (called by :class:`Future`).
"""
ev = StateChangedEvent(state)
if self.thread() is QThread.currentThread():
QCoreApplication.sendEvent(self, ev)
else:
QCoreApplication.postEvent(self, ev)
def setBioMartRegistry(self, registry):
assert(QThread.currentThread() is self.thread())
self.setEnabled(True)
self.registry = registry
self.marts = [mart for mart in self.registry.marts()
if getattr(mart, "visible", "0") != "0"]
self.martsCombo.clear()
for mart in self.marts:
self.martsCombo.addItem(mart.displayName)
def __call__(self, exc_type, exc_value, tb):
if self._stream:
header = exc_type.__name__ + ' Exception'
if QThread.currentThread() != QCoreApplication.instance().thread():
header += " (in non-GUI thread)"
text = traceback.format_exception(exc_type, exc_value, tb)
text.insert(0, '{:-^79}\n'.format(' ' + header + ' '))
text.append('-' * 79 + '\n')
self._stream.writelines(text)
self.handledException.emit((exc_type, exc_value, tb))
def setStatusMessage(self, text):
"""
Set widget's status message.
This is a short status string to be displayed inline next to
the instantiated widget icon in the canvas.
"""
assert QThread.currentThread() == self.thread()
if self.__statusMessage != text:
self.__statusMessage = text
self.statusMessageChanged.emit(text)
def flush(self):
"""
Flush all pending signal emits currently enqueued.
Must only ever be called from the thread this object lives in
(:func:`QObject.thread()`).
"""
if QThread.currentThread() is not self.thread():
raise RuntimeError("`flush()` called from a wrong thread.")
# NOTE: QEvent.MetaCall is the event implementing the
# `Qt.QueuedConnection` method invocation.
QCoreApplication.sendPostedEvents(self, QEvent.MetaCall)
def __on_exception(self, idx, ex):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
if isinstance(ex, NotEnoughData):
self.Error.not_enough_data(len(self.data), self.k_from + idx)
# Only show failed message if there is only 1 k to compute
elif not self.optimize_k:
self.Error.failed(str(ex))
self.clusterings[self.k_from + idx] = str(ex)
def test_executor(self):
executor = ThreadExecutor()
f = executor.submit(QThread.currentThread)
self.assertIsNot(f.result(3), QThread.currentThread())
f = executor.submit(lambda: 1 / 0)
with self.assertRaises(ZeroDivisionError):
f.result()
results = []
task = Task(function=QThread.currentThread)
task.resultReady.connect(results.append, Qt.DirectConnection)
f = executor.submit(task)
self.assertIsNot(f.result(3), QThread.currentThread())
executor.shutdown()
def _initializemodel(self):
assert self.thread() is QThread.currentThread()
model, self.gds_info, self.gds = self._inittask.result()
model.setParent(self)
proxy = self.treeWidget.model()
proxy.setFilterKeyColumn(0)
proxy.setFilterRole(TextFilterRole)
proxy.setFilterCaseSensitivity(False)
proxy.setFilterFixedString(self.filterString)
proxy.setSourceModel(model)
proxy.sort(0, Qt.DescendingOrder)
self.progressBarFinished()
self.setBlocking(False)
self.setEnabled(True)
filter_items = " ".join(
gds[key] for gds in self.gds for key in self.searchKeys
)
tr_chars = ",.:;!?(){}[]_-+\\|/%#@$^&*<>~`"
tr_table = str.maketrans(tr_chars, " " * len(tr_chars))
filter_items = filter_items.translate(tr_table)
filter_items = sorted(set(filter_items.split(" ")))
filter_items = [item for item in filter_items if len(item) > 3]
self.completer.setTokenList(filter_items)
if self.currentGds:
current_id = self.currentGds["dataset_id"]
gdss = [(i, qunpack(proxy.data(proxy.index(i, 1), Qt.DisplayRole)))
for i in range(proxy.rowCount())]
current = [i for i, data in gdss if data and data == current_id]
if current:
current_index = proxy.index(current[0], 0)
self.treeWidget.selectionModel().select(
current_index,
QItemSelectionModel.Select | QItemSelectionModel.Rows
)
self.treeWidget.scrollTo(
current_index, QTreeView.PositionAtCenter)
for i in range(8):
self.treeWidget.resizeColumnToContents(i)
self.treeWidget.setColumnWidth(
1, min(self.treeWidget.columnWidth(1), 300))
self.treeWidget.setColumnWidth(
2, min(self.treeWidget.columnWidth(2), 200))
self.updateInfo()
def test_executor(self):
executor = ThreadExecutor()
f = executor.submit(QThread.currentThread)
self.assertIsNot(f.result(3), QThread.currentThread())
f = executor.submit(lambda: 1 / 0)
with self.assertRaises(ZeroDivisionError):
f.result()
results = []
task = Task(function=QThread.currentThread)
task.resultReady.connect(results.append, Qt.DirectConnection)
f = executor.submit_task(task)
self.assertIsNot(f.result(3), QThread.currentThread())
executor.shutdown()
def __on_exception(self, idx, ex):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
if isinstance(ex, NotEnoughData):
self.Error.not_enough_data(len(self.data), self.k_from + idx)
# Only show failed message if there is only 1 k to compute
elif not self.optimize_k:
self.Error.failed(str(ex))
self.clusterings[self.k_from + idx] = str(ex)
def test_task(self):
results = []
task = Task(function=QThread.currentThread)
task.resultReady.connect(results.append)
task.start()
self.app.processEvents()
self.assertSequenceEqual(results, [QThread.currentThread()])
results = []
thread = QThread()
thread.start()
task = Task(function=QThread.currentThread)
task.moveToThread(thread)
self.assertIsNot(task.thread(), QThread.currentThread())
self.assertIs(task.thread(), thread)
task.resultReady.connect(results.append, Qt.DirectConnection)
task.start()
f = task.future()
self.assertIsNot(f.result(3), QThread.currentThread())
self.assertIs(f.result(3), results[-1])
def onDownloadFinished(self):
# on download completed/canceled/error
assert QThread.currentThread() is self.thread()
for task in list(self._tasks):
future = task.future()
if future.done():
self.EndDownloadTask(task)
self._tasks.remove(task)
if not self._tasks:
# Clear/reset the overall progress
self.progress.setRange(0, 0)
self.cancelButton.setEnabled(False)
def onResultsReady(self, results):
"""Handle the results of centering and z-scoring
"""
assert(QThread.currentThread() is self.thread())
Gc, Rc, z_scores = results
self.setEnabled(True)
self.setBlocking(False)
self.progressBarFinished()
self.centered = Gc, Rc
self.z_scores = z_scores
self.plotMA(Gc, Rc, z_scores, self.zCutoff)
self.commitIf()
def test_qinvoke(self):
executor = ThreadPoolExecutor()
state = [None, None] # type: List[Optional[QThread]]
class StateSetter(QObject):
didsetstate = Signal()
def set_state(self, value: QThread) -> None:
state[0] = value
state[1] = QThread.currentThread()
self.didsetstate.emit()
def func(callback): # type: (Callable[[QThread], None]) -> None
callback(QThread.currentThread())
obj = StateSetter()
spy = QSignalSpy(obj.didsetstate)
f = executor.submit(
func, qinvoke(obj.set_state, context=obj)
)
self.assertTrue(spy.wait())
self.assertIs(state[1], QThread.currentThread(),
"set_state was called from the wrong thread")
self.assertIsNot(state[0], QThread.currentThread(),
"set_state was invoked in the main thread")
# test that disconnect works 'atomically' w.r.t. event loop
spy = QSignalSpy(obj.didsetstate)
callback = qinvoke(obj.set_state, context=obj)
f = executor.submit(func, callback)
f.result()
time.sleep(0.01) # wait for finish
callback.disconnect() # type: ignore
self.assertFalse(spy.wait(100), "")
self.assertSequenceEqual(spy, [])
executor.shutdown(wait=True)
def __init__(self, parent=None):
QObject.__init__(self, parent)
assert QThread.currentThread() is QApplication.instance().thread()
netmanager = self._NETMANAGER_REF and self._NETMANAGER_REF()
if netmanager is None:
netmanager = QNetworkAccessManager()
cache = QNetworkDiskCache()
cache.setCacheDirectory(
os.path.join(data_dir(), "geo", __name__ + ".GeoMap.Cache"))
netmanager.setCache(cache)
ImageLoader._NETMANAGER_REF = weakref.ref(netmanager)
self._netmanager = netmanager
def onDownloadFinished(self):
# on download completed/canceled/error
assert QThread.currentThread() is self.thread()
for task in list(self._tasks):
future = task.future()
if future.done():
self.EndDownloadTask(task)
self._tasks.remove(task)
if not self._tasks:
# Clear/reset the overall progress
self.progress.setRange(0, 0)
self.cancelButton.setEnabled(False)
def onResultsReady(self, results):
"""Handle the results of centering and z-scoring
"""
assert (QThread.currentThread() is self.thread())
Gc, Rc, z_scores = results
self.setEnabled(True)
self.setBlocking(False)
self.progressBarFinished()
self.centered = Gc, Rc
self.z_scores = z_scores
self.plotMA(Gc, Rc, z_scores, self.zCutoff)
self.commitIf()
def _initializemodel(self):
assert self.thread() is QThread.currentThread()
model, self.gds_info, self.gds = self._inittask.result()
model.setParent(self)
proxy = self.treeWidget.model()
proxy.setFilterKeyColumn(0)
proxy.setFilterRole(TextFilterRole)
proxy.setFilterCaseSensitivity(False)
proxy.setFilterFixedString(self.filterString)
proxy.setSourceModel(model)
proxy.sort(0, Qt.DescendingOrder)
self.progressBarFinished()
self.setBlocking(False)
self.setEnabled(True)
filter_items = " ".join(gds[key] for gds in self.gds
for key in self.searchKeys)
tr_chars = ",.:;!?(){}[]_-+\\|/%#@$^&*<>~`"
tr_table = str.maketrans(tr_chars, " " * len(tr_chars))
filter_items = filter_items.translate(tr_table)
filter_items = sorted(set(filter_items.split(" ")))
filter_items = [item for item in filter_items if len(item) > 3]
self.completer.setTokenList(filter_items)
if self.currentGds:
current_id = self.currentGds["dataset_id"]
gdss = [(i, qunpack(proxy.data(proxy.index(i, 1), Qt.DisplayRole)))
for i in range(proxy.rowCount())]
current = [i for i, data in gdss if data and data == current_id]
if current:
current_index = proxy.index(current[0], 0)
self.treeWidget.selectionModel().select(
current_index,
QItemSelectionModel.Select | QItemSelectionModel.Rows)
self.treeWidget.scrollTo(current_index,
QTreeView.PositionAtCenter)
for i in range(8):
self.treeWidget.resizeColumnToContents(i)
self.treeWidget.setColumnWidth(
1, min(self.treeWidget.columnWidth(1), 300))
self.treeWidget.setColumnWidth(
2, min(self.treeWidget.columnWidth(2), 200))
self.updateInfo()
def __task_complete(self, task):
# handle a completed task
assert self.thread() is QThread.currentThread()
if self.__task is not task:
assert task.cancelled
log.debug("Reaping cancelled task: %r", "<>")
return
self.setBlocking(False)
self.progressBarFinished(processEvents=None)
self.setStatusMessage("")
result = task.future
assert result.done()
self.__task = None
try:
results = result.result() # type: Results
learners = results.learners # type: List[Learner]
except Exception as er:
log.exception("testing error (in __task_complete):",
exc_info=True)
self.error("\n".join(traceback.format_exception_only(type(er), er)))
self.__state = State.Done
return
self.__state = State.Done
learner_key = {slot.learner: key for key, slot in
self.learners.items()}
assert all(learner in learner_key for learner in learners)
# Update the results for individual learners
class_var = results.domain.class_var
for learner, result in zip(learners, results.split_by_model()):
stats = None
if class_var.is_primitive():
ex = result.failed[0]
if ex:
stats = [Try.Fail(ex)] * len(self.scorers)
result = Try.Fail(ex)
else:
stats = [Try(scorer_caller(scorer, result))
for scorer in self.scorers]
result = Try.Success(result)
key = learner_key.get(learner)
self.learners[key] = \
self.learners[key]._replace(results=result, stats=stats)
self._update_header()
self._update_stats_model()
self.commit()
def __make_task_runnable(self, task):
if task.thread() is not QThread.currentThread():
raise ValueError("Can only submit Tasks from it's own " +
"thread.")
if task.parent() is not None:
raise ValueError("Can not submit Tasks with a parent.")
task.moveToThread(self._get_depot_thread())
# Use the Task's own Future object
f = task.future()
runnable = _TaskRunnable(f, task, (), {})
return f, runnable
def __make_task_runnable(self, task):
if task.thread() is not QThread.currentThread():
raise ValueError("Can only submit Tasks from it's own " +
"thread.")
if task.parent() is not None:
raise ValueError("Can not submit Tasks with a parent.")
task.moveToThread(self._get_depot_thread())
# Use the Task's own Future object
f = task.future()
runnable = _TaskRunnable(f, task, (), {})
return f, runnable
def _end_task(self, f):
assert self.thread() is QThread.currentThread()
assert threading.current_thread() == threading.main_thread()
assert self._task is not None
assert self._task.future is f
assert f.done()
self._task = None
self.parent.progressBarFinished()
self.parent.filter_genes()
try:
f.result()
except Exception as ex:
raise ex
def __init__(self, parent=None):
QObject.__init__(self, parent)
assert QThread.currentThread() is QApplication.instance().thread()
netmanager = self._NETMANAGER_REF and self._NETMANAGER_REF()
if netmanager is None:
netmanager = QNetworkAccessManager()
cache = QNetworkDiskCache()
cache.setCacheDirectory(
os.path.join(settings.widget_settings_dir(),
__name__ + ".ImageLoader.Cache")
)
netmanager.setCache(cache)
ImageLoader._NETMANAGER_REF = weakref.ref(netmanager)
self._netmanager = netmanager
def _task_finished(self, f):
"""
Parameters
----------
f : Future
The future instance holding the result of learner evaluation.
"""
assert self.thread() is QThread.currentThread()
assert self._task is not None
assert self._task.future is f
assert f.done()
self._task = None
self.progressBarFinished()
try:
results = f.result() # type: List[Results]
except Exception as ex:
# Log the exception with a traceback
log = logging.getLogger()
log.exception(__name__, exc_info=True)
self.error("Exception occurred during evaluation: {!r}".format(ex))
# clear all results
for key in self.results.keys():
self.results[key] = None
else:
# split the combined result into per learner/model results ...
results = [
list(Results.split_by_model(p_results))
for p_results in results
] # type: List[List[Results]]
assert all(len(r.learners) == 1 for r1 in results for r in r1)
assert len(results) == len(self.curvePoints)
learners = [r.learners[0] for r in results[0]]
learner_id = {
learner: id_
for id_, learner in self.learners.items()
}
# ... and update self.results
for i, learner in enumerate(learners):
id_ = learner_id[learner]
self.results[id_] = [p_results[i] for p_results in results]
# [end-snippet-9]
# update the display
self._update_curve_points()
self._update_table()
def __task_complete(self, f: 'Future[Results]'):
# handle a completed task
assert self.thread() is QThread.currentThread()
assert self.__task is not None and self.__task.future is f
self.progressBarFinished()
self.setStatusMessage("")
assert f.done()
self.__task = None
self.__state = State.Done
try:
results = f.result() # type: Results
learners = results.learners # type: List[Learner]
except Exception as er: # pylint: disable=broad-except
log.exception("testing error (in __task_complete):", exc_info=True)
self.error("\n".join(traceback.format_exception_only(type(er),
er)))
return
learner_key = {
slot.learner: key
for key, slot in self.learners.items()
}
assert all(learner in learner_key for learner in learners)
# Update the results for individual learners
class_var = results.domain.class_var
for learner, result in zip(learners, results.split_by_model()):
stats = None
if class_var.is_primitive():
ex = result.failed[0]
if ex:
stats = [Try.Fail(ex)] * len(self.scorers)
result = Try.Fail(ex)
else:
stats = [
Try(scorer_caller(scorer, result))
for scorer in self.scorers
]
result = Try.Success(result)
key = learner_key.get(learner)
self.learners[key] = \
self.learners[key]._replace(results=result, stats=stats)
self.score_table.update_header(self.scorers)
self.update_stats_model()
self.update_comparison_table()
self.commit()
def test_executor(self):
executor = ThreadExecutor()
f1 = executor.submit(pow, 100, 100)
f2 = executor.submit(lambda: 1 / 0)
f3 = executor.submit(QThread.currentThread)
self.assertTrue(f1.result(), pow(100, 100))
with self.assertRaises(ZeroDivisionError):
f2.result()
self.assertIsInstance(f2.exception(), ZeroDivisionError)
self.assertIsNot(f3.result(), QThread.currentThread())
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
corpus, errors, lemmas, pos, ner, is_conllu = None, [], None, None, \
None, False
try:
corpus, errors, lemmas, pos, ner, is_conllu = task.future.result()
except NoDocumentsException:
state = State.Error
self.error("Folder contains no readable files.")
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if corpus:
self.n_text_data = len(corpus)
self.n_text_categories = len(corpus.domain.class_var.values) \
if corpus.domain.class_var else 0
self.base_corpus = self.corpus = corpus
self.is_conllu = is_conllu
self.tokens = lemmas
self.pos = pos
self.ner = ner
if self.corpus:
self.corpus.name = "Documents"
self.skipped_documents = errors
if len(errors):
self.Warning.read_error(
"Some files" if len(errors) > 1 else "One file"
)
self.__setRuntimeState(state)
self.commit()
def run_more_jobs(self):
known = set(self.known)
needed = self.needed_methods()
for method in needed:
if method.name not in self.running_jobs:
setattr(self, "lbl_" + method.name, "pending")
doable = [method for method in needed
if method.name not in self.running_jobs
and set(method.args) <= known]
free = max(1, QThread.idealThreadCount()) - len(self.running_jobs)
for method in doable[:free]:
job = WorkerThread(method, self.known)
job.finished.connect(lambda job=job: self.job_finished(job))
self.running_jobs[method.name] = job
job.start()
if not method.level == INTERNAL:
setattr(self, "lbl_" + method.name, "running")
self.show_computing()
def __commit_finished(self):
assert QThread.currentThread() is self.thread()
assert self.__task is not None
assert self.data is not None
self.__task = None
self.setBlocking(False)
self.progressBarFinished()
if self.optimize_k:
self.update_results()
if self.optimize_k and all(isinstance(self.clusterings[i], str)
for i in range(self.k_from, self.k_to + 1)):
# Show the error of the last clustering
self.Error.failed(self.clusterings[self.k_to])
self.send_data()
def setBioMartConfiguration(self, configuration):
assert (QThread.currentThread() is self.thread())
self.setEnabled(True)
# parse the xml in the main thread (a long time ago this step was
# done in a thread but would frequently cause `expat` to segfault.
doc = biomart.parseXML(io.BytesIO(configuration))
config = list(doc.elements("DatasetConfig"))[0]
configuration = biomart.DatasetConfig(self.registry, config.tag,
config.attributes,
config.children)
self.clearConfiguration()
self.configuration = configuration
def hidden(tree):
return getattr(tree, "hidden", "false") != "false" or \
getattr(tree, "hideDisplay", "false") != "false"
self.attributePagesTabWidget = tabs = gui.tabWidget(
self.attributesConfigurationBox)
for page in configuration.elements("AttributePage"):
if not hidden(page):
page_widget = PageWidget(page, self.dataset, self)
gui.createTabPage(tabs,
getattr(page, "displayName", ""),
widgetToAdd=page_widget,
canScroll=True)
if self.SHOW_FILTERS:
self.filterPagesTabWidget = tabs = gui.tabWidget(
self.filtersConfigurationBox)
for page in configuration.elements("FilterPage"):
if not hidden(page):
page_widget = PageWidget(page, self.dataset, self)
gui.createTabPage(tabs,
getattr(page, "displayName", ""),
widgetToAdd=page_widget,
canScroll=True)
self.afterInit()
self.commitButton.setEnabled(True)
def __set_index(self, f):
# type: (Future) -> None
# set results from `list_remote` query.
assert QThread.currentThread() is self.thread()
assert f.done()
self.setBlocking(False)
self.setStatusMessage("")
self.allinfo_local = self.list_local()
try:
self.allinfo_remote = f.result(
) # type: Dict[Tuple[str, ...], dict]
except Exception:
log.exception("Error while fetching updated index")
if not self.allinfo_local:
self.Error.no_remote_datasets()
else:
self.Warning.only_local_datasets()
self.allinfo_remote = {}
model, current_index = self.create_model()
self.view.model().setSourceModel(model)
self.view.selectionModel().selectionChanged.connect(
self.__on_selection)
self.view.resizeColumnToContents(0)
self.view.setColumnWidth(
1,
min(self.view.sizeHintForColumn(1),
self.view.fontMetrics().width("X" * 24)))
header = self.view.header()
header.restoreState(self.header_state)
# Update the info text
self.infolabel.setText(
format_info(model.rowCount(), len(self.allinfo_local)))
if current_index != -1:
selmodel = self.view.selectionModel()
selmodel.select(
self.view.model().mapFromSource(model.index(current_index, 0)),
QItemSelectionModel.ClearAndSelect | QItemSelectionModel.Rows)
def __call__(self, exc_type, exc_value, tb):
if self.stream is None:
stream = sys.stderr
else:
stream = self.stream
if stream is not None:
header = exc_type.__name__ + ' Exception'
if QThread.currentThread() != QCoreApplication.instance().thread():
header += " (in non-GUI thread)"
text = traceback.format_exception(exc_type, exc_value, tb)
text.insert(0, '{:-^79}\n'.format(' ' + header + ' '))
text.append('-' * 79 + '\n')
stream.writelines(text)
try:
stream.flush()
except Exception:
pass
self.handledException.emit((exc_type, exc_value, tb))
def __set_index(self, f):
# type: (Future) -> None
# set results from `list_remote` query.
assert QThread.currentThread() is self.thread()
assert f.done()
self.setBlocking(False)
self.setStatusMessage("")
self.allinfo_local = list_local(self.local_cache_path)
try:
self.allinfo_remote = f.result()
except Exception: # anytying can happen, pylint: disable=broad-except
log.exception("Error while fetching updated index")
if not self.allinfo_local:
self.Error.no_remote_datasets()
else:
self.Warning.only_local_datasets()
self.allinfo_remote = {}
model, current_index = self.create_model()
self.view.model().setSourceModel(model)
self.view.selectionModel().selectionChanged.connect(
self.__on_selection)
scw = self.view.setColumnWidth
width = self.view.fontMetrics().width
self.view.resizeColumnToContents(0)
scw(self.Header.title, width("X" * 37))
scw(self.Header.size,
20 + max(width("888 bytes "), width("9999.9 MB ")))
scw(self.Header.instances, 20 + width("100000000"))
scw(self.Header.variables, 20 + width("1000000"))
header = self.view.header()
header.restoreState(self.header_state)
if current_index != -1:
selmodel = self.view.selectionModel()
selmodel.select(
self.view.model().mapFromSource(model.index(current_index, 0)),
QItemSelectionModel.ClearAndSelect | QItemSelectionModel.Rows)
self.commit()
def _init_gene_sets_finished(self, f):
assert self.thread() is QThread.currentThread()
assert threading.current_thread() == threading.main_thread()
assert self._task is not None
assert self._task.future is f
assert f.done()
self._task = None
self.progress_bar.finish()
self.setStatusMessage('')
try:
results = f.result() # type: list
[self.data_model.appendRow(model_item) for model_item in results]
self.filter_proxy_model.setSourceModel(self.data_model)
self._update_fdr()
self.filter_data_view()
except Exception as ex:
print(ex)
def __set_results(self, f):
assert self.thread() is QThread.currentThread()
if self._task is None or self._task.future is not f:
self._log.info("Reaping stale task")
return
assert f.done()
task, self._task = self._task, None
self.auto_commit_widget.setDisabled(False)
self.cancel_button.setDisabled(True)
self.cb_image_attr.setDisabled(False)
self.cb_embedder.setDisabled(False)
self.progressBarFinished(processEvents=None)
self.setBlocking(False)
try:
embeddings = f.result()
except ConnectionError:
self._log.exception("Error", exc_info=True)
self.Outputs.embeddings.send(None)
self.Outputs.skipped_images.send(None)
self._set_server_info(connected=False)
return
except Exception as err:
self._log.exception("Error", exc_info=True)
self.error(
"\n".join(traceback.format_exception_only(type(err), err)))
self.Outputs.embeddings.send(None)
self.Outputs.skipped_images.send(None)
return
assert self._input_data is not None
assert len(self._input_data) == len(task.file_paths_mask)
# Missing paths/urls were filtered out. Restore the full embeddings
# array from information stored in task.file_path_mask ...
embeddings_all = [None] * len(task.file_paths_mask)
for i, embedding in zip(np.flatnonzero(~task.file_paths_mask),
embeddings):
embeddings_all[i] = embedding
embeddings_all = np.array(embeddings_all)
self._send_output_signals(embeddings_all)
def node_info_complete(self, future):
assert self.thread() is QThread.currentThread()
assert future.done()
self._task = None
self.progressBarFinished()
self.setCursor(Qt.ArrowCursor)
self.btn_connect.setText(self.LABEL_CONNECT)
self.Information.fetching_node_info.clear()
try:
lst, min_time, max_time = future.result(
) # type: Tuple[List[List], pd.Timestamp, pd.Timestamp]
except self.Cancelled:
pass
except Exception as e:
log.exception("Error fetching node info")
self.Error.fetching_node_info_failed(e)
else:
self.model.wrap(lst)
CachedNodeInfoTable.dump_list(lst)
self.btn_download.setEnabled(True)
# Apply saved row selection
if self.selection:
try:
selection = QItemSelection()
for row in self.model.mapFromSourceRows(self.selection):
selection.select(self.model.index(row, 0))
self.view.selectionModel().select(selection)
except Exception:
log.exception('Failed to restore selection')
self.selection = []
self.date_from.setDateTime(
QDateTime.fromMSecsSinceEpoch(min_time.timestamp() * 1000,
Qt.UTC))
self.date_to.setDateTime(
QDateTime.fromMSecsSinceEpoch(max_time.timestamp() * 1000,
Qt.UTC))
def setBlocking(self, state=True):
"""
Set blocking flag for this widget.
While this flag is set this widget and all its descendants
will not receive any new signals from the workflow signal manager.
This is useful for instance if the widget does it's work in a
separate thread or schedules processing from the event queue.
In this case it can set the blocking flag in it's processNewSignals
method schedule the task and return immediately. After the task
has completed the widget can clear the flag and send the updated
outputs.
.. note::
Failure to clear this flag will block dependent nodes forever.
"""
assert QThread.currentThread() is self.thread()
if self.__blocking != state:
self.__blocking = state
self.blockingStateChanged.emit(state)
def __set_results(self, f):
assert self.thread() is QThread.currentThread()
if self._task is None or self._task.future is not f:
self._log.info("Reaping stale task")
return
assert f.done()
task, self._task = self._task, None
self.auto_commit_widget.setDisabled(False)
self.cancel_button.setDisabled(True)
self.cb_smiles_attr.setDisabled(False)
self.cb_embedder.setDisabled(False)
self.progressBarFinished(processEvents=None)
self.setBlocking(False)
try:
embeddings = f.result()
except ConnectionError:
self._log.exception("Error", exc_info=True)
self.send(_Output.FINGERPRINTS, None)
self.send(_Output.SKIPPED_SMILES, None)
self._set_server_info(connected=False)
return
except Exception as err:
self._log.exception("Error", exc_info=True)
self.error("\n".join(
traceback.format_exception_only(type(err), err)))
self.send(_Output.FINGERPRINTS, None)
self.send(_Output.SKIPPED_SMILES, None)
return
assert self._input_data is not None
assert len(self._input_data) == len(task.smiles)
embeddings_all = [None] * len(task.smiles)
for i, embedding in enumerate(embeddings):
embeddings_all[i] = embedding
embeddings_all = np.array(embeddings_all)
self._send_output_signals(embeddings_all)
def _task_finished(self, f):
"""
Parameters
----------
f : Future
The future instance holding the result of learner evaluation.
"""
assert self.thread() is QThread.currentThread()
assert self._task is not None
assert self._task.future is f
assert f.done()
self._task = None
self.progressBarFinished()
self.model.eval() # eval mode (batchnorm uses moving mean/variance instead of mini-batch mean/variance)
with torch.no_grad():
correct = 0
total = 0
for images, labels in self.test_loader:
images = images.to(self.device)
labels = labels.to(self.device)
outputs = self.model(images)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
print('Test Accuracy of the model on the 10000 test images: {} %'.format(100 * correct / total))
# try:
# result = f.result() # type: List[Results]
# except Exception as ex:
# # Log the exception with a traceback
# log = logging.getLogger()
# log.exception(__name__, exc_info=True)
# self.error("Exception occurred during evaluation: {!r}".format(ex))
# # clear all results
# self.result= None
# else:
print(self.learn.validate())
def __commit_complete(self, f):
# complete the commit operation after the required file has been
# downloaded
assert QThread.currentThread() is self.thread()
assert self.__awaiting_state is not None
assert self.__awaiting_state.future is f
if self.isBlocking():
self.progressBarFinished()
self.setBlocking(False)
self.setStatusMessage("")
self.__awaiting_state = None
try:
path = f.result()
# anything can happen here, pylint: disable=broad-except
except Exception as ex:
log.exception("Error:")
self.error(format_exception(ex))
path = None
self.load_and_output(path)
def get_event_loop() -> asyncio.AbstractEventLoop:
"""
Get the asyncio.AbstractEventLoop for the main Qt application thread.
The QCoreApplication instance must already have been created.
Must only be called from the main Qt application thread.
"""
try:
# Python >= 3.7
get_running_loop = asyncio.get_running_loop # type: ignore
except AttributeError:
get_running_loop = asyncio._get_running_loop # type: ignore
app = QCoreApplication.instance()
if app is None:
raise RuntimeError("QCoreApplication is not running")
if app.thread() is not QThread.currentThread():
raise RuntimeError("Called from non-main thread")
loop: Optional[asyncio.AbstractEventLoop]
try:
loop = get_running_loop()
except RuntimeError:
loop = None
else:
if loop is not None:
return loop
qt_api = os.environ.get("QT_API", "").lower()
if qt_api == "pyqt5":
os.environ["QT_API"] = "PyQt5"
elif qt_api == "pyside2":
os.environ["QT_API"] = "PySide2"
import qasync
if loop is None:
loop = qasync.QEventLoop(app)
# Do not use qasync.QEventLoop's default executor which uses QThread
# based pool and exhibits https://github.com/numpy/numpy/issues/11551
loop.set_default_executor(futures.ThreadPoolExecutor())
return loop
def __onRunFinished(self):
assert QThread.currentThread() is self.thread()
assert self.__state == State.Processing
assert self.__pendingTask is not None
assert self.sender() is self.__pendingTask.watcher
assert self.__pendingTask.future.done()
task = self.__pendingTask
self.__pendingTask = None
try:
corpus, errors = task.future.result()
except Exception:
sys.excepthook(*sys.exc_info())
state = State.Error
corpus = None
errors = []
self.error(traceback.format_exc())
else:
state = State.Done
self.error()
if corpus:
self.n_text_data = len(corpus)
self.n_text_categories = len(corpus.domain.class_var.values)\
if corpus.domain.class_var else 0
self.corpus = corpus
self.corpus.name = "Documents"
self.skipped_documents = errors
if len(errors):
self.Warning.read_error(
"Some files" if len(errors) > 1 else "One file"
)
self.__setRuntimeState(state)
self.commit()
def _fetch_indicators(self, progress=lambda val: None):
"""Background task for fetching indicators."""
progress(0)
def row_item(display_value, item_values=None):
"""Generate a cell item for a given row."""
if not item_values:
item_values = {}
item = QtGui.QStandardItem()
item.setData(display_value, Qt.DisplayRole)
for role, value in item_values.items():
item.setData(value, role)
return item
progress(10)
filter_ = self._main_widget.basic_indicator_filter()
data = self._api.get_indicators(filter_=filter_)
self._indicator_data = {ind["id"]: ind for ind in data}
progress(70)
indicators = [[""] + row
for row in self._api.get_indicator_list(filter_=filter_)]
model = QtGui.QStandardItemModel()
model.setHorizontalHeaderLabels(indicators[0])
for row in indicators[1:]:
search_string = " | ".join(row).lower()
row_data = [row_item(item) for item in row]
row_data[0].setData(search_string, TEXTFILTERROLE)
row_data[1].setData(self._get_link(row[1]), gui.LinkRole)
model.appendRow(row_data)
progress(100)
if QThread.currentThread() is not QCoreApplication.instance().thread():
model.moveToThread(QCoreApplication.instance().thread())
return model
