def test_watcher(self):
executor = ThreadPoolExecutor(max_workers=1)
f = executor.submit(lambda: 42)
w = FutureWatcher(f)
def spies(w):
return SimpleNamespace(
done=QSignalSpy(w.done),
finished=QSignalSpy(w.finished),
result=QSignalSpy(w.resultReady),
error=QSignalSpy(w.exceptionReady),
cancelled=QSignalSpy(w.cancelled),
)
spy = spies(w)
self.assertTrue(spy.done.wait())
self.assertEqual(list(spy.done), [[f]])
self.assertEqual(list(spy.finished), [[f]])
self.assertEqual(list(spy.result), [[42]])
self.assertEqual(list(spy.error), [])
self.assertEqual(list(spy.cancelled), [])
f = executor.submit(lambda: 1 / 0)
w = FutureWatcher(f)
spy = spies(w)
self.assertTrue(spy.done.wait())
self.assertEqual(list(spy.done), [[f]])
self.assertEqual(list(spy.finished), [[f]])
self.assertEqual(len(spy.error), 1)
self.assertIsInstance(spy.error[0][0], ZeroDivisionError)
self.assertEqual(list(spy.result), [])
self.assertEqual(list(spy.cancelled), [])
ev = threading.Event()
# block the executor to test cancellation
executor.submit(lambda: ev.wait())
f = executor.submit(lambda: 0)
w = FutureWatcher(f)
self.assertTrue(f.cancel())
ev.set()
spy = spies(w)
self.assertTrue(spy.done.wait())
self.assertEqual(list(spy.done), [[f]])
self.assertEqual(list(spy.finished), [])
self.assertEqual(list(spy.error), [])
self.assertEqual(list(spy.result), [])
self.assertEqual(list(spy.cancelled), [[f]])
def init_gene_sets(self):
if self._task is not None:
self.cancel()
assert self._task is None
self._task = Task()
self.init_item_model()
# save setting on selected hierarchies
self.stored_gene_sets_selection = self.gs_widget.get_hierarchies(only_selected=True)
f = self.create_partial()
progress_iterations = sum(
(
len(g_set)
for hier, g_set in self.gs_widget.gs_object.map_hierarchy_to_sets().items()
if hier in self.stored_gene_sets_selection
)
)
self.progress_bar = ProgressBar(self, iterations=progress_iterations)
self._task.future = self._executor.submit(f)
self._task.watcher = FutureWatcher(self._task.future)
self._task.watcher.done.connect(self._init_gene_sets_finished)
def score_genes(self, **kwargs):
""" Run gene enrichment.
:param design:
:param data_x:
:param rows_by_cluster:
:param method:
Note:
We do not apply filter nor notify view that data is changed. This is done after filters
"""
if self._task is not None:
# First make sure any pending tasks are cancelled.
self.cancel()
assert self._task is None
progress_advance = methodinvoke(self, "progress_advance", (bool,))
def callback():
if self._task.cancelled:
raise KeyboardInterrupt()
progress_advance(self._task.cancelled)
self.parent.progress_bar = ProgressBar(self.parent, iterations=len(self.get_rows()))
f = partial(self._score_genes, callback=callback, **kwargs)
self._task = Task()
self._task.future = self._executor.submit(f)
self._task.watcher = FutureWatcher(self._task.future)
self._task.watcher.done.connect(self._end_task)
def __init__(self, *args):
super().__init__(*args)
self.__future = None
self.watcher = FutureWatcher()
self.__interuption_requested = False
self.__progress = 0
# Helpers to route the signal emits via a this object's queue.
# This ensures 'atomic' disconnect from signals for targets/slots
# in the same thread. Requires that the event loop is running in this
# object's thread.
self._p_status_changed.connect(self.status_changed,
Qt.QueuedConnection)
self._p_progress_changed.connect(self.progress_changed,
Qt.QueuedConnection)
self._p_partial_result_ready.connect(self.partial_result_ready,
Qt.QueuedConnection)
def compute(self):
if not self.chosen_X:
self.Error.no_vars_selected()
return
if not self.chosen_y:
self.Error.no_class_selected()
return
self.btn_compute.setEnabled(False)
yvar = self.data.domain[self.chosen_y]
def get_col(var, col):
values = np.array(list(var.values) + [np.nan], dtype=object)
pd.Categorical(col, list(var.values))
col = pd.Series(col).fillna(-1).astype(int)
return values[col]
X = np.column_stack([
get_col(var,
self.data.get_column_view(var)[0])
for var in (self.data.domain[i] for i in self.chosen_X)
])
X = pd.DataFrame(X, columns=self.chosen_X)
y = pd.Series(self.data.get_column_view(yvar)[0])
test, args, kwargs = None, (X, y), dict(min_count=self.min_count)
if self.is_permutation:
statistic = 'chi2' if yvar.is_discrete else self.TEST_STATISTICS[
self.test_statistic]
test = perm_test
kwargs.update(statistic=statistic,
n_jobs=-2,
callback=methodinvoke(self, "setProgressValue",
(int, int)))
else:
if yvar.is_discrete:
if len(yvar.values) > 2:
test = chi2_test
else:
test = hyper_test
args = (X, y.astype(bool))
else:
test = {
'mean': t_test,
'variance': fligner_killeen_test,
'median': mannwhitneyu_test,
'minimum': gumbel_min_test,
'maximum': gumbel_max_test,
}[self.test_statistic]
self._task = task = self.Task()
self.progressBarInit()
task.future = self._executor.submit(test, *args, **kwargs)
task.watcher = FutureWatcher(task.future)
task.watcher.done.connect(self.on_computed)
def run_task(self, slug, func):
if self._task is not None:
self.cancel()
assert self._task is None
self.progressBarInit()
self._task = ResolweTask(slug)
self._task.future = self._executor.submit(func)
self._task.watcher = FutureWatcher(self._task.future)
self._task.watcher.done.connect(self.task_finished)
def download(self):
if self._task is not None:
self.cancel()
return
self.btn_connect.setEnabled(False)
self.setCursor(Qt.BusyCursor)
self.btn_download.setText(self.LABEL_CANCEL)
# Remember selected tables
layout = self.box_include_data.layout()
tables = self.included_data = {
w.text()
for w in (layout.itemAt(i).widget() for i in range(layout.count()))
if w.isChecked()
}
# Remeber selected rows
rows = [mi.row() for mi in self.view.selectionModel().selectedRows(0)]
self.selection = rows
nodes = [
self.model[row][0] for row in self.model.mapToSourceRows(rows)
]
self._task = task = self.Task()
self.progressBarInit()
set_progress = methodinvoke(self, "setProgressValue", (float, ))
def func():
if not nodes:
raise self.Cancelled
def progress_advance(*_, _counter=0):
if task.cancelled:
raise self.Cancelled
_counter += 1
set_progress(_counter / (len(tables) + 1))
df = getdf(
tables,
nodeid=nodes,
start_time=self.date_from.textFromDateTime(
self.date_from.dateTime()),
end_time=self.date_to.textFromDateTime(
self.date_to.dateTime()),
freq=self.RESOLUTION[self.sample_resolution],
interpolate=self.INTERPOLATION[self.sample_interpolation],
callback=progress_advance)
return df
task.future = self._executor.submit(func)
task.watcher = FutureWatcher(task.future)
task.watcher.done.connect(self.download_complete)
def _update(self):
if self._task is not None:
# First make sure any pending tasks are cancelled.
self.cancel()
assert self._task is None
if self.train_data is None and self.test_data is None:
return
# collect all learners for which results have not yet been computed
if self.model is None:
return
# setup the task state
self._task = task = Task()
# The learning_curve[_with_test_data] also takes a callback function
# to report the progress. We instrument this callback to both invoke
# the appropriate slots on this widget for reporting the progress
# (in a thread safe manner) and to implement cooperative cancellation.
set_progress = methodinvoke(self, "setProgressValue", (float, ))
def callback(finished, loss):
# check if the task has been cancelled and raise an exception
# from within. This 'strategy' can only be used with code that
# properly cleans up after itself in the case of an exception
# (does not leave any global locks, opened file descriptors, ...)
if task.cancelled:
raise KeyboardInterrupt()
set_progress(finished * 100)
self.output_results.append([self.index, loss])
self.index += 1
# self.Outputs.losses.send(Table.from_list(domain, loss_list))
self.progressBarInit()
# Submit the evaluation function to the executor and fill in the
# task with the resultant Future.
# task.future = self._executor.submit(self.learn.fit_one_cycle(1))
fit_model = partial(train_model,
self.model,
self.epochs,
self.train_data,
self.test_data,
self.device,
self.criterion,
self.optimizer,
self.train_button,
callback=callback)
task.future = self._executor.submit(fit_model)
# Setup the FutureWatcher to notify us of completion
task.watcher = FutureWatcher(task.future)
# by using FutureWatcher we ensure `_task_finished` slot will be
# called from the main GUI thread by the Qt's event loop
task.watcher.done.connect(self._task_finished)
def run_task(self):
if self._task is not None:
self.cancel()
assert self._task is None
self.progressBarInit()
func = partial(self.res.download_data_table, self.data_table_object)
self._task = ResolweTask('download')
self._task.future = self._executor.submit(func)
self._task.watcher = FutureWatcher(self._task.future)
self._task.watcher.done.connect(self.task_finished)
def run_task(self, slug, func):
if self._task is not None:
try:
self.cancel()
except CancelledError as e:
print(e)
assert self._task is None
self.progressBarInit()
self._task = ResolweTask(slug)
self._task.future = self._executor.submit(func)
self._task.watcher = FutureWatcher(self._task.future)
self._task.watcher.finished.connect(self.task_finished)
def __start_download(self, files_list):
# type: (List[Tuple[str, str]]) -> None
task = EnsureDownloaded(files_list)
task.progress.connect(self._progressBarSet)
f = self._executor.submit(task)
fw = FutureWatcher(f, self)
fw.finished.connect(self.__download_finish)
fw.finished.connect(fw.deleteLater)
fw.resultReady.connect(self.__invalidate)
self.progressBarInit(processEvents=None)
self.setBlocking(True)
self.setStatusMessage("Downloading")
self.__state = State.Downloading
def commit_calc(self):
num_nan = np.count_nonzero(np.isnan(self.to_explain.X[0]))
self.to_explain = self.to_explain.transform(self.data.domain)
if num_nan != np.count_nonzero(np.isnan(self.to_explain.X[0])):
self.Warning.unknowns_increased()
if self.model is not None:
# calculate contributions
if self.e is None:
self.e = ExplainPredictions(self.data,
self.model,
batch_size=min(
len(self.data.X), 500),
p_val=self.gui_p_val,
error=self.gui_error)
self._task = task = Task()
def callback(progress):
nonlocal task
# update progress bar
QMetaObject.invokeMethod(self, "set_progress_value",
Qt.QueuedConnection,
Q_ARG(int, progress))
if task.canceled:
return True
return False
def callback_update(table):
QMetaObject.invokeMethod(self, "update_view",
Qt.QueuedConnection,
Q_ARG(Orange.data.Table, table))
def callback_prediction(class_value):
QMetaObject.invokeMethod(self, "update_model_prediction",
Qt.QueuedConnection,
Q_ARG(float, class_value))
self.was_canceled = False
explain_func = partial(self.e.anytime_explain,
self.to_explain[0],
callback=callback,
update_func=callback_update,
update_prediction=callback_prediction)
self.progressBarInit(processEvents=None)
task.future = self._executor.submit(explain_func)
task.watcher = FutureWatcher(task.future)
task.watcher.done.connect(self._task_finished)
self.cancel_button.setDisabled(False)
def __cancel_task(self, wait=True):
# Cancel and dispose of the current task
if self.__task is not None:
state, self.__task = self.__task, None
state.cancel()
state.partial_result_ready.disconnect(self.__set_partial_results)
state.status_changed.disconnect(self.setStatusMessage)
state.progress_changed.disconnect(self.progressBarSet)
state.watcher.done.disconnect(self.__on_done)
if wait:
futures.wait([state.future])
state.deleteLater()
else:
w = FutureWatcher(state.future, parent=state)
w.done.connect(state.deleteLater)
def __init__(self, *args):
super().__init__(*args)
self.__future = None
self.watcher = FutureWatcher()
self.__interuption_requested = False
self.__progress = 0
# Helpers to route the signal emits via a this object's queue.
# This ensures 'atomic' disconnect from signals for targets/slots
# in the same thread. Requires that the event loop is running in this
# object's thread.
self._p_status_changed.connect(
self.status_changed, Qt.QueuedConnection)
self._p_progress_changed.connect(
self.progress_changed, Qt.QueuedConnection)
self._p_partial_result_ready.connect(
self.partial_result_ready, Qt.QueuedConnection)
def init_gene_sets(self, reference_genes=None):
if self._task is not None:
self.cancel()
assert self._task is None
self._task = Task()
progress_advance = methodinvoke(self, "progress_advance")
def callback():
if self._task.cancelled:
raise KeyboardInterrupt()
if self.progress_bar:
progress_advance()
if reference_genes is None:
reference_genes = self.gene_sets_obj.genes()
self.init_item_model()
sets_to_display = self.get_hierarchies(only_selected=True)
# save setting on selected hierarchies
self.stored_selections = sets_to_display
# save context
self.closeContext()
f = partial(self.set_items,
self.gene_sets_obj,
sets_to_display,
set(self.input_genes),
reference_genes,
self.min_count if self.use_min_count else 1,
callback=callback)
progress_iterations = sum([
len(g_set) for hier, g_set in
self.gene_sets_obj.map_hierarchy_to_sets().items()
if hier in sets_to_display
])
self.progress_bar = ProgressBar(self, iterations=progress_iterations)
self._task.future = self._executor.submit(f)
self._task.watcher = FutureWatcher(self._task.future)
self._task.watcher.done.connect(self._init_gene_sets_finished)
self.openContext(self.tax_id)
def _start_task_init(self, f):
if self._task is not None:
self.cancel()
assert self._task is None
self._task = Task("init")
def callback(finished):
if self._task.cancelled:
raise KeyboardInterrupt()
self.progressBarSet(finished * 50)
f = partial(f, callback=callback)
self.progressBarInit()
self._task.future = self._executor.submit(f)
self._task.watcher = FutureWatcher(self._task.future)
self._task.watcher.done.connect(self._init_task_finished)
def async_call(self, func):
self._task = task = Task()
# set_progress = methodinvoke(self, "setProgressValue", (float,))
# def callback(finished):
# # check if the task has been cancelled and raise an exception
# # from within. This 'strategy' can only be used with code that
# # properly cleans up after itself in the case of an exception
# # (does not leave any global locks, opened file descriptors, ...)
# if task.cancelled:
# raise KeyboardInterrupt()
# set_progress(finished * 100)
# _apply_func = partial(_apply_func, callback=callback)
self.progressBarInit()
# Submit the evaluation function to the executor and fill in the
# task with the resultant Future.
task.future = self._executor.submit(func)
# Setup the FutureWatcher to notify us of completion
task.watcher = FutureWatcher(task.future)
# by using FutureWatcher we ensure `_task_finished` slot will be
# called from the main GUI thread by the Qt's event loop
task.watcher.done.connect(self._task_finished)
def _update(self):
if self._task is not None:
# First make sure any pending tasks are cancelled.
self.cancel()
assert self._task is None
if self.data is None:
return
# collect all learners for which results have not yet been computed
need_update = [(id, learner) for id, learner in self.learners.items()
if self.results[id] is None]
if not need_update:
return
# [end-snippet-5]
# [start-snippet-6]
learners = [learner for _, learner in need_update]
# setup the learner evaluations as partial function capturing
# the necessary arguments.
if self.testdata is None:
learning_curve_func = partial(
learning_curve,
learners,
self.data,
folds=self.folds,
proportions=self.curvePoints,
)
else:
learning_curve_func = partial(
learning_curve_with_test_data,
learners,
self.data,
self.testdata,
times=self.folds,
proportions=self.curvePoints,
)
# [end-snippet-6]
# [start-snippet-7]
# setup the task state
self._task = task = Task()
# The learning_curve[_with_test_data] also takes a callback function
# to report the progress. We instrument this callback to both invoke
# the appropriate slots on this widget for reporting the progress
# (in a thread safe manner) and to implement cooperative cancellation.
set_progress = methodinvoke(self, "setProgressValue", (float, ))
def callback(finished):
# check if the task has been cancelled and raise an exception
# from within. This 'strategy' can only be used with code that
# properly cleans up after itself in the case of an exception
# (does not leave any global locks, opened file descriptors, ...)
if task.cancelled:
raise KeyboardInterrupt()
set_progress(finished * 100)
# capture the callback in the partial function
learning_curve_func = partial(learning_curve_func, callback=callback)
# [end-snippet-7]
# [start-snippet-8]
self.progressBarInit()
# Submit the evaluation function to the executor and fill in the
# task with the resultant Future.
task.future = self._executor.submit(learning_curve_func)
# Setup the FutureWatcher to notify us of completion
task.watcher = FutureWatcher(task.future)
# by using FutureWatcher we ensure `_task_finished` slot will be
# called from the main GUI thread by the Qt's event loop
task.watcher.done.connect(self._task_finished)
class TaskState(QObject):
status_changed = Signal(str)
_p_status_changed = Signal(str)
progress_changed = Signal(float)
_p_progress_changed = Signal(float)
partial_result_ready = Signal(object)
_p_partial_result_ready = Signal(object)
def __init__(self, *args):
super().__init__(*args)
self.__future = None
self.watcher = FutureWatcher()
self.__interuption_requested = False
self.__progress = 0
# Helpers to route the signal emits via a this object's queue.
# This ensures 'atomic' disconnect from signals for targets/slots
# in the same thread. Requires that the event loop is running in this
# object's thread.
self._p_status_changed.connect(self.status_changed,
Qt.QueuedConnection)
self._p_progress_changed.connect(self.progress_changed,
Qt.QueuedConnection)
self._p_partial_result_ready.connect(self.partial_result_ready,
Qt.QueuedConnection)
@property
def future(self):
# type: () -> Future
return self.__future
def set_status(self, text):
self._p_status_changed.emit(text)
def set_progress_value(self, value):
if round(value, 1) > round(self.__progress, 1):
# Only emit progress when it has changed sufficiently
self._p_progress_changed.emit(value)
self.__progress = value
def set_partial_results(self, value):
self._p_partial_result_ready.emit(value)
def is_interuption_requested(self):
return self.__interuption_requested
def start(self, executor, func=None):
# type: (futures.Executor, Callable[[], Any]) -> Future
assert self.future is None
assert not self.__interuption_requested
self.__future = executor.submit(func)
self.watcher.setFuture(self.future)
return self.future
def cancel(self):
assert not self.__interuption_requested
self.__interuption_requested = True
if self.future is not None:
rval = self.future.cancel()
else:
# not even scheduled yet
rval = True
return rval
def start(self):
"""
Start/execute the image indexing operation
"""
self.error()
self.__invalidated = False
if self.currentPath is None:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("Starting a new task while one is in progress. "
"Cancel the existing task (dir:'{}')".format(
self.__pendingTask.startdir))
self.cancel()
startdir = self.currentPath
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(self, "__onReportProgress", (object, ))
task = ImportImages(report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=3)
except UserInterruptError:
pass
except TimeoutError:
log.info("The task did not stop in in a timely manner")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_image_scan_task_interupt():
try:
return task(startdir)
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_image_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
def setFuture(self, future):
if self.future is not None:
raise RuntimeError("future is already set")
self.future = future
self.watcher = FutureWatcher(future, parent=self)
self.watcher.done.connect(self.done)
def start(self):
"""
Start/execute the text indexing operation
"""
self.error()
self.Warning.clear()
self.progress_widget.setValue(0)
self.__invalidated = False
startdir = self.currentPath if self.source == self.LOCAL_FILE \
else self.url_combo.currentText().strip()
if not startdir:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("Starting a new task while one is in progress. "
"Cancel the existing task (dir:'{}')"
.format(self.__pendingTask.startdir))
self.cancel()
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(
self, "__onReportProgress", (object,))
task = ImportDocuments(startdir, self.source == self.URL,
report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=0)
except UserInterruptError:
pass
except TimeoutError:
log.info("The task did not stop in in a timely manner")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_text_scan_task_interupt():
try:
return task.run()
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_text_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
def Enrichment(self):
assert self.input_data is not None
assert self.__state == State.Ready
if not self.annotations.ontology:
self.annotations.ontology = self.ontology
self.error(1)
self.warning([0, 1])
self.__get_input_genes()
self.input_genes = set(self.input_genes)
self.known_input_genes = self.annotations.get_genes_with_known_annotation(self.input_genes)
# self.clusterGenes = clusterGenes = self.annotations.map_to_ncbi_id(self.input_genes).values()
self.infoLabel.setText("%i unique genes on input\n%i (%.1f%%) genes with known annotations" %
(len(self.input_genes), len(self.known_input_genes),
100.0*len(self.known_input_genes)/len(self.input_genes)
if len(self.input_genes) else 0.0))
if not self.useReferenceDataset or self.ref_data is None:
self.information(2)
self.information(1)
self.ref_genes = self.annotations.genes()
self.ref_genes = set(self.ref_genes)
elif self.ref_data is not None:
self.__get_ref_genes()
self.ref_genes = set(self.ref_genes)
ref_count = len(self.ref_genes)
if ref_count == 0:
self.ref_genes = self.annotations.genes()
self.referenceRadioBox.buttons[1].setText("Reference set")
self.referenceRadioBox.buttons[1].setDisabled(True)
self.information(2, "Unable to extract gene names from reference dataset. "
"Using entire genome for reference")
self.useReferenceDataset = 0
else:
self.referenceRadioBox.buttons[1].setText("Reference set ({} genes)".format(ref_count))
self.referenceRadioBox.buttons[1].setDisabled(False)
self.information(2)
else:
self.useReferenceDataset = 0
self.ref_genes = []
if not self.ref_genes:
self.error(1, "No valid reference set")
return {}
evidences = []
for etype in go.evidenceTypesOrdered:
if self.useEvidenceType[etype]:
evidences.append(etype)
aspect = ['Process', 'Component', 'Function'][self.aspectIndex]
self.progressBarInit(processEvents=False)
self.setBlocking(True)
self.__state = State.Running
if self.input_genes:
f = self._executor.submit(
self.annotations.get_enriched_terms,
self.input_genes, self.ref_genes, evidences, aspect=aspect,
prob=self.probFunctions[self.probFunc], use_fdr=False,
progress_callback=methodinvoke(
self, "_progressBarSet", (float,))
)
fw = FutureWatcher(f, parent=self)
fw.done.connect(self.__on_enrichment_done)
fw.done.connect(fw.deleteLater)
return
else:
f = Future()
f.set_result({})
self.__on_enrichment_done(f)
def start(self):
"""
Start/execute the text indexing operation
"""
self.error()
self.Warning.clear()
self.progress_widget.setValue(0)
self.__invalidated = False
if self.currentPath is None:
return
if self.__state == State.Processing:
assert self.__pendingTask is not None
log.info("启动新任务，另一个任务正在运行中 "
"取消已有的任务 (dir:'{}')".format(self.__pendingTask.startdir))
self.cancel()
startdir = self.currentPath
self.__setRuntimeState(State.Processing)
report_progress = methodinvoke(self, "__onReportProgress", (object, ))
task = ImportDocuments(startdir, report_progress=report_progress)
# collect the task state in one convenient place
self.__pendingTask = taskstate = namespace(
task=task,
startdir=startdir,
future=None,
watcher=None,
cancelled=False,
cancel=None,
)
def cancel():
# Cancel the task and disconnect
if taskstate.future.cancel():
pass
else:
taskstate.task.cancelled = True
taskstate.cancelled = True
try:
taskstate.future.result(timeout=0)
except UserInterruptError:
pass
except TimeoutError:
log.info("任务未及时停止")
taskstate.watcher.finished.disconnect(self.__onRunFinished)
taskstate.cancel = cancel
def run_text_scan_task_interupt():
try:
return task.run()
except UserInterruptError:
# Suppress interrupt errors, so they are not logged
return
taskstate.future = self.__executor.submit(run_text_scan_task_interupt)
taskstate.watcher = FutureWatcher(taskstate.future)
taskstate.watcher.finished.connect(self.__onRunFinished)
class TaskState(QObject):
status_changed = Signal(str)
_p_status_changed = Signal(str)
progress_changed = Signal(float)
_p_progress_changed = Signal(float)
partial_result_ready = Signal(object)
_p_partial_result_ready = Signal(object)
def __init__(self, *args):
super().__init__(*args)
self.__future = None
self.watcher = FutureWatcher()
self.__interuption_requested = False
self.__progress = 0
# Helpers to route the signal emits via a this object's queue.
# This ensures 'atomic' disconnect from signals for targets/slots
# in the same thread. Requires that the event loop is running in this
# object's thread.
self._p_status_changed.connect(
self.status_changed, Qt.QueuedConnection)
self._p_progress_changed.connect(
self.progress_changed, Qt.QueuedConnection)
self._p_partial_result_ready.connect(
self.partial_result_ready, Qt.QueuedConnection)
@property
def future(self):
# type: () -> Future
return self.__future
def set_status(self, text):
self._p_status_changed.emit(text)
def set_progress_value(self, value):
if round(value, 1) > round(self.__progress, 1):
# Only emit progress when it has changed sufficiently
self._p_progress_changed.emit(value)
self.__progress = value
def set_partial_results(self, value):
self._p_partial_result_ready.emit(value)
def is_interuption_requested(self):
return self.__interuption_requested
def start(self, executor, func=None):
# type: (futures.Executor, Callable[[], Any]) -> Future
assert self.future is None
assert not self.__interuption_requested
self.__future = executor.submit(func)
self.watcher.setFuture(self.future)
return self.future
def cancel(self):
assert not self.__interuption_requested
self.__interuption_requested = True
if self.future is not None:
rval = self.future.cancel()
else:
# not even scheduled yet
rval = True
return rval
def Enrichment(self):
assert self.clusterDataset is not None
assert self.__state == State.Ready
if not self.annotations.ontology:
self.annotations.ontology = self.ontology
if isinstance(self.annotations.genematcher, gene.GMDirect):
self.SetGeneMatcher()
self.error(1)
self.warning([0, 1])
if self.useAttrNames:
clusterGenes = [
v.name for v in self.clusterDataset.domain.attributes
]
self.information(0)
elif 0 <= self.geneAttrIndex < len(self.candidateGeneAttrs):
geneAttr = self.candidateGeneAttrs[self.geneAttrIndex]
clusterGenes = [
str(ex[geneAttr]) for ex in self.clusterDataset
if not numpy.isnan(ex[geneAttr])
]
if any("," in gene for gene in clusterGenes):
self.information(
0,
"Separators detected in cluster gene names. Assuming multiple genes per example."
)
clusterGenes = reduce(operator.iadd,
(genes.split(",")
for genes in clusterGenes), [])
else:
self.information(0)
else:
self.error(1, "Failed to extract gene names from input dataset!")
return {}
genesSetCount = len(set(clusterGenes))
self.clusterGenes = clusterGenes = self.annotations.get_gene_names_translator(
clusterGenes).values()
self.infoLabel.setText(
"%i unique genes on input\n%i (%.1f%%) genes with known annotations"
% (genesSetCount, len(clusterGenes), 100.0 * len(clusterGenes) /
genesSetCount if genesSetCount else 0.0))
referenceGenes = None
if not self.useReferenceDataset or self.referenceDataset is None:
self.information(2)
self.information(1)
referenceGenes = self.annotations.gene_names
elif self.referenceDataset is not None:
if self.useAttrNames:
referenceGenes = [
v.name for v in self.referenceDataset.domain.attributes
]
self.information(1)
elif geneAttr in (self.referenceDataset.domain.variables +
self.referenceDataset.domain.metas):
referenceGenes = [
str(ex[geneAttr]) for ex in self.referenceDataset
if not numpy.isnan(ex[geneAttr])
]
if any("," in gene for gene in clusterGenes):
self.information(
1,
"Separators detected in reference gene names. Assuming multiple genes per example."
)
referenceGenes = reduce(operator.iadd,
(genes.split(",")
for genes in referenceGenes), [])
else:
self.information(1)
else:
self.information(1)
referenceGenes = None
if referenceGenes is None:
referenceGenes = list(self.annotations.gene_names)
self.referenceRadioBox.buttons[1].setText("Reference set")
self.referenceRadioBox.buttons[1].setDisabled(True)
self.information(
2,
"Unable to extract gene names from reference dataset. Using entire genome for reference"
)
self.useReferenceDataset = 0
else:
refc = len(referenceGenes)
referenceGenes = self.annotations.get_gene_names_translator(
referenceGenes).values()
self.referenceRadioBox.buttons[1].setText(
"Reference set (%i genes, %i matched)" %
(refc, len(referenceGenes)))
self.referenceRadioBox.buttons[1].setDisabled(False)
self.information(2)
else:
self.useReferenceDataset = 0
if not referenceGenes:
self.error(1, "No valid reference set")
return {}
self.referenceGenes = referenceGenes
evidences = []
for etype in go.evidenceTypesOrdered:
if self.useEvidenceType[etype]:
evidences.append(etype)
aspect = ["P", "C", "F"][self.aspectIndex]
self.progressBarInit(processEvents=False)
self.setBlocking(True)
self.__state = State.Running
if clusterGenes:
f = self._executor.submit(self.annotations.get_enriched_terms,
clusterGenes,
referenceGenes,
evidences,
aspect=aspect,
prob=self.probFunctions[self.probFunc],
use_fdr=False,
progress_callback=methodinvoke(
self, "_progressBarSet", (float, )))
fw = FutureWatcher(f, parent=self)
fw.done.connect(self.__on_enrichment_done)
fw.done.connect(fw.deleteLater)
return
else:
f = Future()
f.set_result({})
self.__on_enrichment_done(f)
def load_data(self):
# Make sure any pending tasks are cancelled.
if self._task is not None:
self.cancel()
return
self.btn_download.setEnabled(False)
self.setCursor(Qt.BusyCursor)
self.btn_connect.setText(self.LABEL_CANCEL)
self.Information.fetching_node_info()
self.Error.fetching_node_info_failed.clear()
CachedNodeInfoTable.clear()
set_connection_params(self.con_hostname,
self.con_port,
self.con_username,
self.con_password,
self.con_database,
timeout=self.con_timeout)
lst = [[node] + [''] * 4 for node in all_nodes()]
# Add tables checkboxes into Download section
layout = self.box_include_data.layout()
while True: # Purge existing
item = layout.takeAt(0)
if not item:
break
item.widget().setParent(None)
for table in all_tables():
layout.addWidget(
QCheckBox(table, self, checked=table in self.included_data))
CachedNodeInfoTable.dump_tables(all_tables())
set_progress = methodinvoke(self, "setProgressValue", (float, ))
self._task = task = self.Task()
self.progressBarInit()
def func():
FREQ = '30m'
min_time = pd.Timestamp('2100-01-01')
max_time = pd.Timestamp('1800-01-01')
for i, row in enumerate(lst):
if task.cancelled:
raise self.Cancelled
node = row[0]
tables = tables_for_node(node)
times = np.array([
table_timerange(table, node, freq=FREQ) for table in tables
])
tmin = times[:, 0].min().tz_convert(
None) # tz_convert strips '+00:00'
tmax = times[:, 1].max().tz_convert(None)
min_time = min(min_time, tmin)
max_time = max(max_time, tmax)
row[0] = int(node)
row[1] = ', '.join(
sorted(
distinct_values('modem',
'Interface',
nodeid=node,
freq=FREQ)))
row[2] = '' if pd.isnull(tmin) else tmin.strftime(
self.PYDATE_FORMAT)
row[3] = '' if pd.isnull(tmax) else tmax.strftime(
self.PYDATE_FORMAT)
row[4] = ', '.join(tables)
set_progress(i / len(lst) * 99)
return lst, min_time, max_time
task.future = self._executor.submit(func)
task.watcher = FutureWatcher(task.future)
task.watcher.done.connect(self.node_info_complete)
