def _finish_task(self): self._client_callback_queue.put_nowait(SIGNAL_TASK_FINISH()) if self._external_callback_queue: self._external_callback_queue.queue.put_nowait( SIGNAL_TASK_FINISH( Signal(id=self._external_callback_queue.id)))
def _failed_task(self, exception): self.__exception_signal.put_nowait(SIGNAL_EXCEPTION(exception)) if self._external_callback_queue: self._external_callback_queue.queue.put_nowait( SIGNAL_EXCEPTION( Signal(id=self._external_callback_queue.id, content=exception)))
def __stop_handler(self): self.file.close() self._acum_time.stop() self._inst_maker.stop() self.runflag = False self._client_callback_queue.put_nowait(SIGNAL_CALLBACK_END()) self.__exception_signal.put_nowait(SIGNAL_TASK_STOP()) if self.__callback_queue: self.__callback_queue.queue.put_nowait( SIGNAL_TASK_STOP(Signal(id=self.__callback_queue.id)))
def _open(self, task_id): wrapper = None dlreq_wrapper = self._full_queue[task_id] handler = self._open_handle(child_process=dlreq_wrapper.child_process, **self._open_kwargs) if dlreq_wrapper.child_process: self._process_collector[task_id] = handler._process open_callback = dlreq_wrapper.callback self._thread_collector.put(self._opening_control_thread, GROUP_OPEN_CTRL, args=(task_id, handler), owner=task_id).start() try: handler.open(dlreq_wrapper.source) except (HandlerError, URLError, ClientError) as e: open_callback.put_nowait( SIGNAL_TASK_FAIL(Signal(id=task_id, content=e))) except Exception as e: open_callback.put_nowait( SIGNAL_TASK_FAIL(Signal(id=task_id, content=e))) raise else: if handler.is_opened(): if dlreq_wrapper.child_process: queue = self._process_queue else: queue = self._thread_queue wrapper = HandlerWrapper( id=task_id, source=handler, child_process=dlreq_wrapper.child_process) handler.install_external_callback_queue( SignalQueue(id=task_id, queue=queue)) wrapper.move_from(dlreq_wrapper) open_callback.put_nowait(SIGNAL_TASK_OPEN(content=wrapper)) else: open_callback.put_nowait(SIGNAL_TASK_PAUSE(Signal(id=task_id))) return wrapper
def __pause_handler(self): """ Callback handler when signal.id == ID_PAUSE: <PAUSE SIGNAL> """ for thread in self._tpm.get_group(GROUP_CLIENT): thread.owner.send_signal(SIGNAL_TASK_PAUSE()) self._tpm.wait(GROUP_SLICER) self._tpm.wait(GROUP_CLIENT) self._tpm.wait(GROUP_RELEASE) self._client_callback_queue.put_nowait(SIGNAL_TASK_STOP()) if self._external_callback_queue: self._external_callback_queue.queue.put_nowait( SIGNAL_TASK_PAUSE(Signal(id=self._external_callback_queue.id)))
def _send_ctrl_signal(self, signal_wrapper, task_id, content=None): signal = signal_wrapper(Signal(id=task_id, content=content)) if self._full_queue[task_id].child_process: self._process_queue.put_nowait(signal) else: self._thread_queue.put_nowait(signal)
from six.moves.queue import Queue as ThreadQueue from multiprocessing import Manager as ProcessManager from nbdler.struct.signal import ID_TASK_FINISH, ID_TASK_STOP, ID_TASK_EXCEPTION, ID_TASK_PAUSE, \ SIGNAL_TASK_PAUSE, Signal, ID_TASK_OPEN, SIGNAL_TASK_OPEN, SIGNAL_TASK_START, ID_TASK_START, \ SIGNAL_TASK_FAIL, ID_TASK_FAIL, SIGNAL_CALLBACK_END, ID_CALLBACK_END from nbdler.struct.misc import SignalQueue, ProcessInfo from nbdler.utils.process_interface import make_class, make_method from nbdler.utils.thread import ThreadCollector, Lock from nbdler.struct.task import TaskWrapper, TaskQueue from nbdler.handler import dlopen from nbdler.exception import HandlerError, URLError, ClientError, \ ChildProcessDisable __all__ = ['Manager', 'manager', 'ManagerForChildProcess'] SIGNAL_CALLBACK_END = SIGNAL_CALLBACK_END(Signal(id=-1)) DEFAULT_MAX_TASK = 2 def manager(max_task=DEFAULT_MAX_TASK, open_args=(), open_kwargs=None, deamon=False, *, child_process=False): if child_process: manager_handler = ManagerForChildProcess else: manager_handler = Manager return manager_handler(max_task, dlopen, open_args, open_kwargs, deamon)