def process_done(future: Future):
nonlocal self
self._tasks_number -= 1
if future.cancelled():
# process_task ended by cancel
self.requeue_message(self.requeue_message(
basic_deliver.delivery_tag)
)
else:
if future.exception():
exception = future.exception()
if (not isinstance(exception, RequeueMessage)
and not isinstance(exception, ChangeQueue)):
self.log.exception(exception)
self.requeue_message(
basic_deliver.delivery_tag
)
if isinstance(exception, ChangeQueue):
if not self.running.is_set():
self.running.clear()
self.log.info("Changing queues")
self.stop_consuming()
self._queue = self.another_queue(
exception.host)
self.running.set()
else:
self.acknowledge_message(basic_deliver.delivery_tag)
def _callback(curr_future: futures.Future,
executor_future: futures.Future):
if executor_future.exception():
curr_future.set_exception(executor_future.exception())
logging.error(executor_future.exception())
else:
curr_future.set_result(executor_future.result())
def wrapped_func(*args, **kwargs):
if threading.current_thread() is threading.main_thread():
return func(*args, **kwargs)
cond = Condition()
future = Future()
def callback_func(dt):
try:
res = func(*args, **kwargs)
with cond:
future.set_result(res)
cond.notify()
except Exception as exc:
with cond:
future.set_exception(exc)
cond.notify()
finally:
with cond:
if not future.done():
future.cancel()
cond.notify()
with cond:
Clock.schedule_once(callback_func, 0)
cond.wait()
assert future.done()
if future.cancelled():
raise Exception("did not execute func %s" % func)
if future.exception():
raise future.exception()
return future.result()
def _check_error(future: Future):
if future.exception() is not None:
if not VERBOSE:
item = str(future.exception()).strip()
else:
item = future.exception()
logger.error('Ошибка: %s', item)
def task_done(future: Future) -> None:
if not task_status_future.done():
if future.cancelled():
task_status_future.cancel()
elif future.exception():
task_status_future.set_exception(future.exception())
else:
exc = RuntimeError('Task exited without calling task_status.started()')
task_status_future.set_exception(exc)
def _schedule_dts_work(self, job_msg):
f = functools.partial(
asyncio.ensure_future,
self._dts_publisher(job_msg),
loop = self.loop)
fut = Future()
self.loop.call_soon_threadsafe(CopyStatusPublisher._async_add, f, fut)
xx = fut.result()
if fut.exception() is not None:
self.log.error("Caught future exception during download: %s type %s", str(fut.exception()), type(fut.exception()))
raise fut.exception()
return xx
def callback(future: Future):
if future.cancelled() or future.exception():
publication_event.state_of_processing = 'FAILURE'
if future.exception():
logger.error('error while producing %s of %s',
publication_event.format_requested,
publication_event.published_object.title(),
exc_info=future.exception())
elif future.done():
publication_event.state_of_processing = 'SUCCESS'
publication_event.save()
def _enter_finished_view(self, future: Future):
mount_tool.umount(self._mounted_usb_dev)
if future.exception() is None:
self.switch(
StatusMessage([
"Success", "The transaction has been signed successfully. "
"The USB stick was automatically unmounted."
], self._backlight))
else:
self.switch(
StatusMessage([
"Error",
"There was an error while signing the transaction: " +
str(future.exception())
], self._backlight))
def _async_core_request(self, method: str, request_params: str,
callback: ResponseHandler) -> Any:
""" Perform core asynchronous request """
# Generate request id
request_id = self._generate_request_id()
# Create concurrent future
future = Future()
# Set response map data
self._async_response_map[request_id] = {
'is_async': False,
'callback': callback,
'future': future
}
# Execute core request
tc_request(ctx=self._client.ctx,
method=method,
request_id=request_id,
params_json=request_params,
response_handler=self._async_response_handler)
# Resolve future
exception = future.exception()
if exception:
raise exception
return future.result()
async def schedthreadedfunc(func, *args, timeout=None, **kwargs):
futr = Future()
t = _thread.start_new_thread(run_future, (func, futr) + args, kwargs)
if timeout is None:
await asyncio.wrap_future(futr)
else:
try:
with async_timeout.timeout(timeout):
await asyncio.wrap_future(futr)
except asyncio.TimeoutError:
ctypes.pythonapi.PyThreadState_SetAsyncExc(
t, ctypes.py_object(TimeoutError))
raise TimeoutError()
if futr.exception():
raise futr.exception()
return futr.result()
def job_stage_callback(job: Job, stage: str, fut: Future) -> None:
''' Generic job stage completion callback.
Accepts a Job model, stage name, and the future.
When this is called, it means the job stage has completed
in some form or fashion. Should use `fut`'s result methods
to figure out what happened and update accordingly.
'''
# First, update job metadata with the execution result.
exc = fut.exception()
if not fut.cancelled() and exc:
raise exc
if fut.done():
job.meta_update(result=fut.result())
job.save()
# Dispatch the next futures chain, if applicable
if stage == STAGE_FETCHING:
# Check if the job has any destinations and trigger the
# destinations executor
if 'destinations' in job.meta_dict:
# Spawn the uploader future
# TODO: Add a marker in the job meta showing
# that destination uploads are queued
job_begin_upload(job)
elif stage == STAGE_UPLOADING:
job.status = 'completed'
job.save()
log.info(f'job {job.id} has finished job pipeline')
def _ready_callback(self, fn, mapping, proxy: Future, future: Future):
""" Internally handles completion of dependencies
"""
with self._lock:
if not proxy in self._pending:
return
del self._pending[proxy]
if future.cancelled():
proxy.cancel()
if not proxy.set_running_or_notify_cancel():
return
exception = future.exception()
if exception is not None:
proxy.set_exception(exception)
return
if mapping is None:
dependencies = future.result()
else:
dependencies = mapping(*future.result())
internal = self._executor.submit(fn, *dependencies)
internal.add_done_callback(partial(self._done_callback, proxy))
def _on_finished(self, future: Future) -> None:
self._syncing = False
gui_hooks.media_sync_did_start_or_stop(False)
exc = future.exception()
if exc is not None:
self._handle_sync_error(exc)
else:
self._log_and_notify(tr(StringsGroup.SYNC, "media-complete"))
def _on_finished(self, future: Future) -> None:
self._syncing = False
gui_hooks.media_sync_did_start_or_stop(False)
exc = future.exception()
if exc is not None:
self._handle_sync_error(exc)
else:
self._log_and_notify(tr(TR.SYNC_MEDIA_COMPLETE))
def _on_finished(self, future: Future) -> None:
hooks.bg_thread_progress_callback.remove(self._on_progress)
self.mw.progress.finish()
self.progress_dialog = None
exc = future.exception()
if isinstance(exc, Interrupted):
return
output: MediaCheckOutput = future.result()
report = output.report
# show report and offer to delete
diag = QDialog(self.mw)
diag.setWindowTitle(tr(TR.MEDIA_CHECK_WINDOW_TITLE))
layout = QVBoxLayout(diag)
diag.setLayout(layout)
text = QTextEdit()
text.setReadOnly(True)
text.setPlainText(report)
layout.addWidget(text)
box = QDialogButtonBox(QDialogButtonBox.Close)
layout.addWidget(box)
if output.unused:
b = QPushButton(tr(TR.MEDIA_CHECK_DELETE_UNUSED))
b.setAutoDefault(False)
box.addButton(b, QDialogButtonBox.RejectRole)
b.clicked.connect(
lambda c: self._on_trash_files(output.unused)) # type: ignore
if output.missing:
if any(map(lambda x: x.startswith("latex-"), output.missing)):
b = QPushButton(tr(TR.MEDIA_CHECK_RENDER_LATEX))
b.setAutoDefault(False)
box.addButton(b, QDialogButtonBox.RejectRole)
b.clicked.connect(self._on_render_latex) # type: ignore
if output.have_trash:
b = QPushButton(tr(TR.MEDIA_CHECK_EMPTY_TRASH))
b.setAutoDefault(False)
box.addButton(b, QDialogButtonBox.RejectRole)
b.clicked.connect(lambda c: self._on_empty_trash()) # type: ignore
b = QPushButton(tr(TR.MEDIA_CHECK_RESTORE_TRASH))
b.setAutoDefault(False)
box.addButton(b, QDialogButtonBox.RejectRole)
b.clicked.connect(
lambda c: self._on_restore_trash()) # type: ignore
box.rejected.connect(diag.reject) # type: ignore
diag.setMinimumHeight(400)
diag.setMinimumWidth(500)
restoreGeom(diag, "checkmediadb")
diag.exec_()
saveGeom(diag, "checkmediadb")
def _on_finished(self, future: Future) -> None:
self._set_progress_enabled(False)
self.mw.progress.finish()
self.progress_dialog = None
exc = future.exception()
if isinstance(exc, Interrupted):
return
output: CheckMediaResponse = future.result()
report = output.report
# show report and offer to delete
diag = QDialog(self.mw)
diag.setWindowTitle(tr.media_check_window_title())
disable_help_button(diag)
layout = QVBoxLayout(diag)
diag.setLayout(layout)
text = QPlainTextEdit()
text.setReadOnly(True)
text.setPlainText(report)
text.setWordWrapMode(QTextOption.WrapMode.NoWrap)
layout.addWidget(text)
box = QDialogButtonBox(QDialogButtonBox.StandardButton.Close)
layout.addWidget(box)
if output.unused:
b = QPushButton(tr.media_check_delete_unused())
b.setAutoDefault(False)
box.addButton(b, QDialogButtonBox.ButtonRole.RejectRole)
qconnect(b.clicked, lambda c: self._on_trash_files(output.unused))
if output.missing:
if any(map(lambda x: x.startswith("latex-"), output.missing)):
b = QPushButton(tr.media_check_render_latex())
b.setAutoDefault(False)
box.addButton(b, QDialogButtonBox.ButtonRole.RejectRole)
qconnect(b.clicked, self._on_render_latex)
if output.have_trash:
b = QPushButton(tr.media_check_empty_trash())
b.setAutoDefault(False)
box.addButton(b, QDialogButtonBox.ButtonRole.RejectRole)
qconnect(b.clicked, lambda c: self._on_empty_trash())
b = QPushButton(tr.media_check_restore_trash())
b.setAutoDefault(False)
box.addButton(b, QDialogButtonBox.ButtonRole.RejectRole)
qconnect(b.clicked, lambda c: self._on_restore_trash())
qconnect(box.rejected, diag.reject)
diag.setMinimumHeight(400)
diag.setMinimumWidth(500)
restoreGeom(diag, "checkmediadb")
diag.exec()
saveGeom(diag, "checkmediadb")
def update_info_received(future: Future):
# if syncing/in profile screen, defer message delivery
if not mgr.mw.col:
mgr.mw.progress.timer(
1000,
lambda: update_info_received(future),
False,
requiresCollection=False,
)
return
if future.exception():
# swallow network errors
print(str(future.exception()))
result = []
else:
result = future.result()
on_done(client, result)
def process_done(future: Future):
nonlocal self
self._tasks_number -= 1
if future.cancelled():
# process_task ended by cancel
self.requeue_message(self.requeue_message(
basic_deliver.delivery_tag)
)
else:
if future.exception():
exception = future.exception()
if not isinstance(exception, RequeueMessage):
self.log.exception(exception)
self.requeue_message(
basic_deliver.delivery_tag
)
else:
self.acknowledge_message(basic_deliver.delivery_tag)
def _user_task_done_callback(self, future: Future, future_set: dict,
task: TaskInfo) -> None:
try:
exc = future.exception()
result = None if exc else future.result()
except (CancelledError, TimeoutError) as loc_exc:
exc = loc_exc
result = None
future_set.pop(future)
task.finish_task(result, exc) # type: ignore
def test_copy_exception(self):
source = Future()
target = Future()
copy(source, target)
source.set_exception(exc)
self.assertFalse(target.cancelled())
self.assertTrue(target.done())
self.assertIs(target.exception(), exc)
def _log_finished(self, f: Future, probe: Probe) -> None:
"""
Logs each safeguard when they terminated.
"""
name = probe.get("name")
x = f.exception()
if x is not None:
logger.debug("Safeguard '{}' failed: {}".format(name, str(x)),
exc_info=x)
else:
logger.debug("Safeguard '{}' finished normally".format(name))
def raise_error_from_future(future: Future):
"""
If you'd lke to log errors that occur in thread pools, call `add_done_callback` on the future returned from
`ThreadPoolExecutor.submit` and pass in this function.
"""
async_task_error: Exception = future.exception()
if async_task_error:
# Using the error_dialog method here seems to just hang the application forever.
# Workaround: log error, make a noise, alert the user, and hope for the best
message = f"An error occurred while running an asynchronous task: <pre>{traceback.format_exc()}</pre>"
print(red_text(message))
def test_copy_success(self):
source = Future()
target = Future()
copy(source, target)
source.set_result(obj)
self.assertFalse(target.cancelled())
self.assertTrue(target.done())
self.assertIsNone(target.exception())
self.assertIs(target.result(), obj)
def completed(f: Future):
exc = f.exception()
event_registry.continous_hypothesis_completed(experiment, journal, exc)
if exc is not None:
if isinstance(exc, InterruptExecution):
journal["status"] = "interrupted"
logger.fatal(str(exc))
elif isinstance(exc, Exception):
journal["status"] = "aborted"
logger.fatal(str(exc))
logger.info("Continous steady state hypothesis terminated")
def _on_task_done(self, future: Future):
assert future.done()
assert self.__task is not None
assert self.__task.future is future
assert self.__task.watcher.future() is future
self.__task, task = None, self.__task
task.deleteLater()
ex = future.exception()
if ex is not None:
self.on_exception(ex)
else:
self.on_done(future.result())
def _on_task_done(self, future: Future):
assert future.done()
assert self.__task is not None
assert self.__task.future is future
assert self.__task.watcher.future() is future
self.__task, task = None, self.__task
task.deleteLater()
ex = future.exception()
if ex is not None:
self.on_exception(ex)
else:
self.on_done(future.result())
def _on_finished(self, future: Future) -> None:
self._syncing = False
if self._progress_timer:
self._progress_timer.stop()
self._progress_timer = None
gui_hooks.media_sync_did_start_or_stop(False)
exc = future.exception()
if exc is not None:
self._handle_sync_error(exc)
else:
self._log_and_notify(tr.sync_media_complete())
def __done(self, f: Future):
"""
Thread task callback, saves result and such
:param f:
:return:
"""
task_id = self.registry[f]
del self.registry[f]
trw = self.load(task_id)
exc = f.exception()
if exc is not None:
trw.error(exc)
else:
trw.closed(f.result())
self.save(trw)
def _done_callback(self, proxy: Future, future: Future):
""" Internally handles completion of executor future, copies result to proxy
Args:
fn (function): [description]
future (Future): [description]
"""
if future.cancelled():
proxy.cancel()
exception = future.exception()
if exception is not None:
proxy.set_exception(exception)
else:
result = future.result()
proxy.set_result(result)
def _future_cb(self, future: Future, handler: Handler):
"""check if error in response and run error-handler"""
err = future.exception()
if err is not None:
# run exception handler
handler.error_received(err)
# print traceback for logs if enabled
if self._kw['print_traceback']:
tb = ''.join(traceback.format_tb(err.__traceback__))
print('Traceback (most recent call last):\n%s%s: %s' % (
''.join(traceback.format_tb(err.__traceback__)),
err.__class__.__name__,
err.args[0],
),
file=sys.stderr)
def callback(prev_future: Future):
if prev_future.cancelled() or not prev_future.done():
next_future.cancel()
return
if prev_future.exception() is not None:
if on_rejected is None:
next_future.set_exception(prev_future.exception())
else:
next_future.set_result(on_rejected(prev_future.exception()))
return
try:
result = prev_future.result()
if on_fulfilled is not None:
result = on_fulfilled(result)
if isinstance(result, Future):
result = result.result()
next_future.set_result(result)
except BaseException as ex:
if on_rejected is None:
next_future.set_exception(ex)
else:
next_future.set_result(on_rejected(ex))
def _on_task_done(self, future: Future):
assert future.done()
assert self.__task is not None
assert self.__task.future is future
assert self.__task.watcher.future() is future
self.__task = None
ex = future.exception()
if ex is not None:
self.on_exception(ex)
else:
self.on_done(future.result())
# This assert prevents user to start new task (call start) from either
# on_done or on_exception
assert self.__task is None, (
"Starting new task from "
f"{'on_done' if ex is None else 'on_exception'} is forbidden")
def _task_complete(
self,
f: futures.Future,
message: Any,
metadata: Optional[MessageMetadata] = None,
):
"""
Clean up after the task and do any necessary logging.
"""
metadata = MessageMetadata(message) if metadata is None else metadata
exception: Optional[Exception] = f.exception()
if exception is not None:
self.logger.error(
"failed processing {message} with the following exception: {exception}"
.format(exception=repr(exception), message=metadata),
exc_info=exception,
)
self.statsd.increment("process.record.failure", 1, tags=[])
else:
self.logger.info(
"successfully processed {message}".format(message=metadata))
self.statsd.increment("process.record.success", 1, tags=[])
self.queue.delete_messages(
Entries=[{
"Id": message.message_id,
"ReceiptHandle": message.receipt_handle,
}])
