def _finish_task(self):
self._client_callback_queue.put_nowait(SIGNAL_TASK_FINISH())
if self._external_callback_queue:
self._external_callback_queue.queue.put_nowait(
SIGNAL_TASK_FINISH(
Signal(id=self._external_callback_queue.id)))
def _failed_task(self, exception):
self.__exception_signal.put_nowait(SIGNAL_EXCEPTION(exception))
if self._external_callback_queue:
self._external_callback_queue.queue.put_nowait(
SIGNAL_EXCEPTION(
Signal(id=self._external_callback_queue.id,
content=exception)))
def __stop_handler(self):
self.file.close()
self._acum_time.stop()
self._inst_maker.stop()
self.runflag = False
self._client_callback_queue.put_nowait(SIGNAL_CALLBACK_END())
self.__exception_signal.put_nowait(SIGNAL_TASK_STOP())
if self.__callback_queue:
self.__callback_queue.queue.put_nowait(
SIGNAL_TASK_STOP(Signal(id=self.__callback_queue.id)))
def _open(self, task_id):
wrapper = None
dlreq_wrapper = self._full_queue[task_id]
handler = self._open_handle(child_process=dlreq_wrapper.child_process,
**self._open_kwargs)
if dlreq_wrapper.child_process:
self._process_collector[task_id] = handler._process
open_callback = dlreq_wrapper.callback
self._thread_collector.put(self._opening_control_thread,
GROUP_OPEN_CTRL,
args=(task_id, handler),
owner=task_id).start()
try:
handler.open(dlreq_wrapper.source)
except (HandlerError, URLError, ClientError) as e:
open_callback.put_nowait(
SIGNAL_TASK_FAIL(Signal(id=task_id, content=e)))
except Exception as e:
open_callback.put_nowait(
SIGNAL_TASK_FAIL(Signal(id=task_id, content=e)))
raise
else:
if handler.is_opened():
if dlreq_wrapper.child_process:
queue = self._process_queue
else:
queue = self._thread_queue
wrapper = HandlerWrapper(
id=task_id,
source=handler,
child_process=dlreq_wrapper.child_process)
handler.install_external_callback_queue(
SignalQueue(id=task_id, queue=queue))
wrapper.move_from(dlreq_wrapper)
open_callback.put_nowait(SIGNAL_TASK_OPEN(content=wrapper))
else:
open_callback.put_nowait(SIGNAL_TASK_PAUSE(Signal(id=task_id)))
return wrapper
def __pause_handler(self):
""" Callback handler when signal.id == ID_PAUSE:
<PAUSE SIGNAL>
"""
for thread in self._tpm.get_group(GROUP_CLIENT):
thread.owner.send_signal(SIGNAL_TASK_PAUSE())
self._tpm.wait(GROUP_SLICER)
self._tpm.wait(GROUP_CLIENT)
self._tpm.wait(GROUP_RELEASE)
self._client_callback_queue.put_nowait(SIGNAL_TASK_STOP())
if self._external_callback_queue:
self._external_callback_queue.queue.put_nowait(
SIGNAL_TASK_PAUSE(Signal(id=self._external_callback_queue.id)))
def _send_ctrl_signal(self, signal_wrapper, task_id, content=None):
signal = signal_wrapper(Signal(id=task_id, content=content))
if self._full_queue[task_id].child_process:
self._process_queue.put_nowait(signal)
else:
self._thread_queue.put_nowait(signal)
from six.moves.queue import Queue as ThreadQueue
from multiprocessing import Manager as ProcessManager
from nbdler.struct.signal import ID_TASK_FINISH, ID_TASK_STOP, ID_TASK_EXCEPTION, ID_TASK_PAUSE, \
SIGNAL_TASK_PAUSE, Signal, ID_TASK_OPEN, SIGNAL_TASK_OPEN, SIGNAL_TASK_START, ID_TASK_START, \
SIGNAL_TASK_FAIL, ID_TASK_FAIL, SIGNAL_CALLBACK_END, ID_CALLBACK_END
from nbdler.struct.misc import SignalQueue, ProcessInfo
from nbdler.utils.process_interface import make_class, make_method
from nbdler.utils.thread import ThreadCollector, Lock
from nbdler.struct.task import TaskWrapper, TaskQueue
from nbdler.handler import dlopen
from nbdler.exception import HandlerError, URLError, ClientError, \
ChildProcessDisable
__all__ = ['Manager', 'manager', 'ManagerForChildProcess']
SIGNAL_CALLBACK_END = SIGNAL_CALLBACK_END(Signal(id=-1))
DEFAULT_MAX_TASK = 2
def manager(max_task=DEFAULT_MAX_TASK,
open_args=(),
open_kwargs=None,
deamon=False,
*,
child_process=False):
if child_process:
manager_handler = ManagerForChildProcess
else:
manager_handler = Manager
return manager_handler(max_task, dlopen, open_args, open_kwargs, deamon)